From cde10526cae8f36cc17d5d5dd1d24629d8d4c75b Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 20 Sep 2024 13:57:51 +0000 Subject: [PATCH 01/44] [Filestore] Implement a tool which replays filestore-vhost's profile log - part0: prepare --- .../libs/diagnostics/events/profile_events.ev | 1 + .../libs/diagnostics/profile_log_events.cpp | 14 ++++++++++++++ cloud/filestore/public/api/protos/node.proto | 1 + 3 files changed, 16 insertions(+) diff --git a/cloud/filestore/libs/diagnostics/events/profile_events.ev b/cloud/filestore/libs/diagnostics/events/profile_events.ev index 5fdab47daf..25f14cfd7f 100644 --- a/cloud/filestore/libs/diagnostics/events/profile_events.ev +++ b/cloud/filestore/libs/diagnostics/events/profile_events.ev @@ -23,6 +23,7 @@ message TProfileLogNodeInfo { optional string NewNodeName = 4; optional uint32 Flags = 5; optional uint32 Mode = 6; + optional uint32 Type = 10; // from response optional uint64 NodeId = 7; diff --git a/cloud/filestore/libs/diagnostics/profile_log_events.cpp b/cloud/filestore/libs/diagnostics/profile_log_events.cpp index 9784a45a0c..076dbfbb65 100644 --- a/cloud/filestore/libs/diagnostics/profile_log_events.cpp +++ b/cloud/filestore/libs/diagnostics/profile_log_events.cpp @@ -282,6 +282,20 @@ void InitProfileLogRequestInfo( auto* nodeInfo = profileLogRequest.MutableNodeInfo(); nodeInfo->SetNewParentNodeId(request.GetNodeId()); nodeInfo->SetNewNodeName(request.GetName()); + + if (request.HasFile()) { + nodeInfo->SetType(NProto::E_REGULAR_NODE); + } else if (request.HasDirectory()) { + nodeInfo->SetType(NProto::E_DIRECTORY_NODE); + } else if (request.HasLink()) { + nodeInfo->SetType(NProto::E_LINK_NODE); + } else if (request.HasSocket()) { + nodeInfo->SetType(NProto::E_SOCK_NODE); + } else if (request.HasSymLink()) { + nodeInfo->SetType(NProto::E_SYMLINK_NODE); + } else { + nodeInfo->SetType(NProto::E_INVALID_NODE); + } } template <> diff --git a/cloud/filestore/public/api/protos/node.proto b/cloud/filestore/public/api/protos/node.proto index 25f5a53621..fcf767a0aa 100644 --- a/cloud/filestore/public/api/protos/node.proto +++ b/cloud/filestore/public/api/protos/node.proto @@ -29,6 +29,7 @@ enum ENodeType E_DIRECTORY_NODE = 2; E_LINK_NODE = 3; E_SOCK_NODE = 4; + E_SYMLINK_NODE = 5; } //////////////////////////////////////////////////////////////////////////////// From c57e0b5df57db2dc1e5c869151a5d072f2d97f25 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 20 Sep 2024 14:09:22 +0000 Subject: [PATCH 02/44] upload dirty --- .../tools/testing/replay/bin/app.cpp | 187 ++ .../filestore/tools/testing/replay/bin/app.h | 14 + .../tools/testing/replay/bin/bootstrap.cpp | 202 ++ .../tools/testing/replay/bin/bootstrap.h | 72 + .../tools/testing/replay/bin/main.cpp | 43 + .../tools/testing/replay/bin/options.cpp | 90 + .../tools/testing/replay/bin/options.h | 40 + .../tools/testing/replay/bin/private.h | 14 + .../tools/testing/replay/bin/ya.make | 29 + .../tools/testing/replay/lib/client.cpp | 1 + .../tools/testing/replay/lib/client.h | 21 + .../tools/testing/replay/lib/context.h | 18 + .../tools/testing/replay/lib/executor.cpp | 60 + .../tools/testing/replay/lib/executor.h | 40 + .../tools/testing/replay/lib/public.h | 20 + .../tools/testing/replay/lib/request.h | 61 + .../tools/testing/replay/lib/request_data.cpp | 639 ++++ .../testing/replay/lib/request_index.cpp | 684 +++++ .../testing/replay/lib/request_replay.cpp | 2678 +++++++++++++++++ .../tools/testing/replay/lib/test.cpp | 1156 +++++++ .../filestore/tools/testing/replay/lib/test.h | 38 + .../tools/testing/replay/lib/ya.make | 36 + .../tools/testing/replay/protos/replay.proto | 125 + .../tools/testing/replay/protos/ya.make | 13 + cloud/filestore/tools/testing/replay/ya.make | 5 + 25 files changed, 6286 insertions(+) create mode 100644 cloud/filestore/tools/testing/replay/bin/app.cpp create mode 100644 cloud/filestore/tools/testing/replay/bin/app.h create mode 100644 cloud/filestore/tools/testing/replay/bin/bootstrap.cpp create mode 100644 cloud/filestore/tools/testing/replay/bin/bootstrap.h create mode 100644 cloud/filestore/tools/testing/replay/bin/main.cpp create mode 100644 cloud/filestore/tools/testing/replay/bin/options.cpp create mode 100644 cloud/filestore/tools/testing/replay/bin/options.h create mode 100644 cloud/filestore/tools/testing/replay/bin/private.h create mode 100644 cloud/filestore/tools/testing/replay/bin/ya.make create mode 100644 cloud/filestore/tools/testing/replay/lib/client.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/client.h create mode 100644 cloud/filestore/tools/testing/replay/lib/context.h create mode 100644 cloud/filestore/tools/testing/replay/lib/executor.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/executor.h create mode 100644 cloud/filestore/tools/testing/replay/lib/public.h create mode 100644 cloud/filestore/tools/testing/replay/lib/request.h create mode 100644 cloud/filestore/tools/testing/replay/lib/request_data.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/request_index.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/request_replay.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/test.cpp create mode 100644 cloud/filestore/tools/testing/replay/lib/test.h create mode 100644 cloud/filestore/tools/testing/replay/lib/ya.make create mode 100644 cloud/filestore/tools/testing/replay/protos/replay.proto create mode 100644 cloud/filestore/tools/testing/replay/protos/ya.make create mode 100644 cloud/filestore/tools/testing/replay/ya.make diff --git a/cloud/filestore/tools/testing/replay/bin/app.cpp b/cloud/filestore/tools/testing/replay/bin/app.cpp new file mode 100644 index 0000000000..707a39132d --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/app.cpp @@ -0,0 +1,187 @@ +#include "app.h" + +#include "bootstrap.h" +#include "options.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace NCloud::NFileStore::NReplay { + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TApp +{ +private: + TLog Log; + + TAppContext Context; + + TMutex Lock; + TMap Tests; + +public: + static TApp* GetInstance() + { + return Singleton(); + } + + int Run(TBootstrap& bootstrap) + { + NCloud::SetCurrentThreadName("main"); + Log = bootstrap.GetLogging()->CreateLog("MAIN"); + + try { + NProto::TTestGraph graph; + + auto options = bootstrap.GetOptions(); + if (options->TestsConfig) { + ParseFromTextFormat(options->TestsConfig, graph); + } + + TVector> tests; + for (ui32 i = 0; i < graph.TestsSize(); ++i) { + const auto& entry = graph.GetTests(i); + + switch (entry.GetTestCase()) { + case NProto::TTestGraph::TTest::kLoadTest: { + auto name = entry.GetLoadTest().GetName(); + + auto test = CreateLoadTest( + Context, + entry.GetLoadTest(), + bootstrap.GetTimer(), + bootstrap.GetScheduler(), + bootstrap.GetLogging(), + bootstrap.GetClientFactory()); + + tests.push_back([=, this]() + { RunLoadTest(i, name, test); }); + + break; + } + default: + Y_ABORT( + "unexpected test case: %s", + entry.ShortDebugString().c_str()); + } + } + + TExecutor executor(Context, std::move(tests)); + executor.Run(); + + for (const auto& pair: Tests) { + bootstrap.GetOptions()->GetOutputStream() + << NProtobufJson::Proto2Json( + pair.second, + {.FormatOutput = true}) + << Endl; + } + + } catch (...) { + STORAGE_ERROR("app has failed: " << CurrentExceptionMessage()); + return 1; + } + + return AtomicGet(Context.ExitCode); + } + + void RunLoadTest(ui32 i, const TString& name, const ITestPtr& test) + { + try { + const auto future = test->Run(); + if (!future.Initialized()) { + return; + } + const auto& stats = future.GetValueSync(); + if (!stats.GetSuccess()) { + ythrow yexception() << "not successful"; + } + + with_lock (Lock) { + Tests[i] = stats; + } + + STORAGE_INFO("%s has finished", MakeTestTag(name).c_str()); + + } catch (...) { + STORAGE_INFO( + "%s test has failed: %s", + MakeTestTag(name).c_str(), + CurrentExceptionMessage().c_str()); + + Stop(1); + } + } + + void Stop(int exitCode) + { + AtomicSet(Context.ExitCode, exitCode); + AtomicSet(Context.ShouldStop, 1); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +void ProcessSignal(int signum) +{ + if (signum == SIGINT || signum == SIGTERM) { + AppStop(0); + } +} + +void ProcessAsyncSignal(int signum) +{ + if (signum == SIGHUP) { + TLogBackend::ReopenAllBackends(); + } +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +void ConfigureSignals() +{ + std::set_new_handler(abort); + + // make sure that errors can be seen by everybody :) + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + + // mask signals + signal(SIGPIPE, SIG_IGN); + signal(SIGINT, ProcessSignal); + signal(SIGTERM, ProcessSignal); + + SetAsyncSignalHandler(SIGHUP, ProcessAsyncSignal); +} + +int AppMain(TBootstrap& bootstrap) +{ + return TApp::GetInstance()->Run(bootstrap); +} + +void AppStop(int exitCode) +{ + TApp::GetInstance()->Stop(exitCode); +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/app.h b/cloud/filestore/tools/testing/replay/bin/app.h new file mode 100644 index 0000000000..bd58a22d72 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/app.h @@ -0,0 +1,14 @@ +#pragma once + +#include "private.h" + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +void ConfigureSignals(); + +int AppMain(TBootstrap& bootstrap); +void AppStop(int exitCode); + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp b/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp new file mode 100644 index 0000000000..415cb3cfa1 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp @@ -0,0 +1,202 @@ +#include "bootstrap.h" + +#include "options.h" + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NMonitoring; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TClientFactory final + : public IClientFactory +{ +private: + const ILoggingServicePtr Logging; + const ITimerPtr Timer; + const ISchedulerPtr Scheduler; + const NClient::TClientConfigPtr ClientConfig; + + TVector Clients; + +public: + TClientFactory( + ILoggingServicePtr logging, + ITimerPtr timer, + ISchedulerPtr scheduler, + NClient::TClientConfigPtr clientConfig) + : Logging(std::move(logging)) + , Timer(std::move(timer)) + , Scheduler(std::move(scheduler)) + , ClientConfig(std::move(clientConfig)) + {} + + void Start() override + { + for (auto& client: Clients) { + client->Start(); + } + } + + void Stop() override + { + for (auto& client: Clients) { + client->Stop(); + } + } + + IFileStoreServicePtr CreateClient() override + { + auto client = NClient::CreateFileStoreClient( + ClientConfig, + Logging); + + client = NClient::CreateDurableClient( + Logging, + Timer, + Scheduler, + CreateRetryPolicy(ClientConfig), + client); + + Clients.push_back(client); + return client; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TBootstrap::TBootstrap(TOptionsPtr options) + : Options(std::move(options)) +{} + +TBootstrap::~TBootstrap() +{} + +void TBootstrap::Init() +{ + InitDbgConfig(); + InitClientConfig(); + + Timer = CreateWallClockTimer(); + Scheduler = CreateScheduler(); + + ClientFactory = std::make_shared( + Logging, + Timer, + Scheduler, + ClientConfig); +} + +void TBootstrap::Start() +{ + if (Logging) { + Logging->Start(); + } + + if (Monitoring) { + Monitoring->Start(); + } + + if (Scheduler) { + Scheduler->Start(); + } + + if (ClientFactory) { + ClientFactory->Start(); + } +} + +void TBootstrap::Stop() +{ + if (ClientFactory) { + ClientFactory->Stop(); + } + + if (Scheduler) { + Scheduler->Stop(); + } + + if (Monitoring) { + Monitoring->Stop(); + } + + if (Logging) { + Logging->Stop(); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +void TBootstrap::InitDbgConfig() +{ + TLogSettings logSettings; + + if (Options->VerboseLevel) { + auto level = GetLogLevel(Options->VerboseLevel); + Y_ENSURE(level, "unknown log level: " << Options->VerboseLevel.Quote()); + logSettings.FiltrationLevel = *level; + } + + Logging = CreateLoggingService("console", logSettings); + Log = Logging->CreateLog("BOOT"); + + if (Options->MonitoringPort) { + Monitoring = CreateMonitoringService( + Options->MonitoringPort, + Options->MonitoringAddress, + Options->MonitoringThreads); + } else { + Monitoring = CreateMonitoringServiceStub(); + } +} + +void TBootstrap::InitClientConfig() +{ + NProto::TClientConfig clientConfig; + + if (Options->ConfigFile) { + ParseFromTextFormat(Options->ConfigFile, clientConfig); + } + if (Options->Host) { + clientConfig.SetHost(Options->Host); + } + if (Options->InsecurePort) { + clientConfig.SetPort(Options->InsecurePort); + } + if (Options->SecurePort) { + clientConfig.SetSecurePort(Options->SecurePort); + } + + ClientConfig = std::make_shared(clientConfig); + + TStringStream ss; + ClientConfig->Dump(ss); + + STORAGE_INFO("client config:\n" << ss.Str()); +} + +} // namespace NCloud::NFileStore::NServer diff --git a/cloud/filestore/tools/testing/replay/bin/bootstrap.h b/cloud/filestore/tools/testing/replay/bin/bootstrap.h new file mode 100644 index 0000000000..e83e4de116 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/bootstrap.h @@ -0,0 +1,72 @@ +#pragma once + +#include "private.h" + +#include + +#include +#include + +#include +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +class TBootstrap +{ +private: + TOptionsPtr Options; + + ILoggingServicePtr Logging; + TLog Log; + + IMonitoringServicePtr Monitoring; + ITimerPtr Timer; + ISchedulerPtr Scheduler; + + NClient::TClientConfigPtr ClientConfig; + IClientFactoryPtr ClientFactory; + +public: + TBootstrap(TOptionsPtr options); + ~TBootstrap(); + + void Init(); + void Start(); + void Stop(); + + TOptionsPtr GetOptions() + { + return Options; + } + + ITimerPtr GetTimer() + { + return Timer; + } + + ISchedulerPtr GetScheduler() + { + return Scheduler; + } + + ILoggingServicePtr GetLogging() + { + return Logging; + } + + IClientFactoryPtr GetClientFactory() + { + return ClientFactory; + } + +private: + void InitDbgConfig(); + void InitClientConfig(); +}; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/main.cpp b/cloud/filestore/tools/testing/replay/bin/main.cpp new file mode 100644 index 0000000000..fa4f75b44d --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/main.cpp @@ -0,0 +1,43 @@ +#include "app.h" +#include "bootstrap.h" +#include "options.h" + +#include + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char** argv) +{ + using namespace NCloud::NFileStore::NReplay; + + ConfigureSignals(); + + auto options = std::make_shared(); + try { + options->Parse(argc, argv); + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } + + TBootstrap bootstrap(std::move(options)); + try { + bootstrap.Init(); + bootstrap.Start(); + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + bootstrap.Stop(); + return 1; + } + + int exitCode = AppMain(bootstrap); + + try { + bootstrap.Stop(); + } catch (...) { + Cerr << CurrentExceptionMessage() << Endl; + return 1; + } + + return exitCode; +} diff --git a/cloud/filestore/tools/testing/replay/bin/options.cpp b/cloud/filestore/tools/testing/replay/bin/options.cpp new file mode 100644 index 0000000000..c7282dd01e --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/options.cpp @@ -0,0 +1,90 @@ +#include "options.h" + +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NLastGetopt; + +//////////////////////////////////////////////////////////////////////////////// + +void TOptions::Parse(int argc, char** argv) +{ + TOpts opts; + opts.AddHelpOption('h'); + + opts.AddLongOption("config", "config file name") + .RequiredArgument("STR") + .StoreResult(&ConfigFile); + + opts.AddLongOption("host", "connect host") + .RequiredArgument("STR") + .StoreResult(&Host); + + opts.AddLongOption("port", "connect port") + .RequiredArgument("NUM") + .StoreResult(&InsecurePort); + + opts.AddLongOption("secure-port", "connect secure port (overrides --port)") + .RequiredArgument("NUM") + .StoreResult(&SecurePort); + + opts.AddLongOption("mon-file") + .RequiredArgument("STR") + .StoreResult(&MonitoringConfig); + + opts.AddLongOption("mon-address") + .RequiredArgument("STR") + .StoreResult(&MonitoringAddress); + + opts.AddLongOption("mon-port") + .RequiredArgument("NUM") + .StoreResult(&MonitoringPort); + + opts.AddLongOption("mon-threads") + .RequiredArgument("NUM") + .StoreResult(&MonitoringThreads); + + opts.AddLongOption("tests-config") + .RequiredArgument("STR") + .StoreResult(&TestsConfig); + + opts.AddLongOption("results") + .RequiredArgument("STR") + .StoreResult(&OutputFile); + + const auto& verbose = opts.AddLongOption("verbose", "output level for diagnostics messages") + .OptionalArgument("STR") + .StoreResult(&VerboseLevel); + + opts.AddLongOption("timeout", "timeout in seconds") + .OptionalArgument("NUM") + .StoreResult(&Timeout); + + opts.AddLongOption("grpc-trace", "turn on grpc tracing") + .NoArgument() + .StoreTrue(&EnableGrpcTracing); + + TOptsParseResultException res(&opts, argc, argv); + + if (res.Has(&verbose) && !VerboseLevel) { + VerboseLevel = "debug"; + } +} + +IOutputStream& TOptions::GetOutputStream() +{ + if (OutputFile && OutputFile != "-") { + if(!OutputStream) { + OutputStream.reset(new TOFStream(OutputFile)); + } + + return *OutputStream; + } + + return Cout; +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/options.h b/cloud/filestore/tools/testing/replay/bin/options.h new file mode 100644 index 0000000000..d31beeee58 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/options.h @@ -0,0 +1,40 @@ +#pragma once + +#include "private.h" + +#include +#include +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct TOptions +{ + TString ConfigFile; + TString Host; + ui32 InsecurePort = 0; + ui32 SecurePort = 0; + + TString MonitoringConfig; + TString MonitoringAddress; + ui32 MonitoringPort = 0; + ui32 MonitoringThreads = 0; + + TString TestsConfig; + + TString OutputFile; + std::unique_ptr OutputStream; + + TDuration Timeout = TDuration::Zero(); + + TString VerboseLevel; + bool EnableGrpcTracing = false; + + void Parse(int argc, char** argv); + + IOutputStream& GetOutputStream(); +}; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/private.h b/cloud/filestore/tools/testing/replay/bin/private.h new file mode 100644 index 0000000000..5dc29567bd --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/private.h @@ -0,0 +1,14 @@ +#pragma once + +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +class TBootstrap; + +struct TOptions; +using TOptionsPtr = std::shared_ptr; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/ya.make b/cloud/filestore/tools/testing/replay/bin/ya.make new file mode 100644 index 0000000000..fe4eba2bca --- /dev/null +++ b/cloud/filestore/tools/testing/replay/bin/ya.make @@ -0,0 +1,29 @@ +PROGRAM(filestore-replay) + +PEERDIR( + cloud/filestore/config + cloud/filestore/public/api/protos + + cloud/filestore/libs/client + cloud/filestore/tools/testing/replay/lib + cloud/filestore/tools/testing/replay/protos + + cloud/storage/core/libs/common + cloud/storage/core/libs/diagnostics + + contrib/ydb/library/actors/util + library/cpp/getopt + library/cpp/logger + library/cpp/protobuf/json + library/cpp/protobuf/util + library/cpp/sighandler +) + +SRCS( + app.cpp + bootstrap.cpp + main.cpp + options.cpp +) + +END() diff --git a/cloud/filestore/tools/testing/replay/lib/client.cpp b/cloud/filestore/tools/testing/replay/lib/client.cpp new file mode 100644 index 0000000000..f679c0d5ec --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/client.cpp @@ -0,0 +1 @@ +#include "client.h" diff --git a/cloud/filestore/tools/testing/replay/lib/client.h b/cloud/filestore/tools/testing/replay/lib/client.h new file mode 100644 index 0000000000..917c506f63 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/client.h @@ -0,0 +1,21 @@ +#pragma once + +#include "public.h" + +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct IClientFactory + : public IStartable +{ + virtual ~IClientFactory() = default; + + virtual IFileStoreServicePtr CreateClient() = 0; +}; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/context.h b/cloud/filestore/tools/testing/replay/lib/context.h new file mode 100644 index 0000000000..2f64b2e974 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/context.h @@ -0,0 +1,18 @@ +#pragma once + +#include "public.h" + +#include +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct TAppContext : TNonCopyable +{ + TAtomic ShouldStop = 0; + TAtomic ExitCode = 0; +}; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/executor.cpp b/cloud/filestore/tools/testing/replay/lib/executor.cpp new file mode 100644 index 0000000000..377de67a8b --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/executor.cpp @@ -0,0 +1,60 @@ +#include "executor.h" + +#include "context.h" + +namespace NCloud::NFileStore::NReplay { + +using namespace NThreading; + +//////////////////////////////////////////////////////////////////////////////// + +TExecutor::TExecutor(const TAppContext& ctx, TActionList actions) + : Context(ctx) + , Actions(std::move(actions)) +{ + ThreadPool.Start(); +} + +void TExecutor::Run() +{ + while (!AtomicGet(Context.ShouldStop)) { + ui32 current = -1; + + with_lock (Lock) { + if (DoneCount == Actions.size()) { + break; + } + + current = DoneCount++; + } + + RunAction(current).Subscribe([=] (auto future) { + future.GetValue(); // re-throw exception + ReadyEvent.Signal(); + }); + + ReadyEvent.WaitI(); + ReadyEvent.Reset(); + } +} + +TFuture TExecutor::RunAction(ui32 action) +{ + auto promise = NewPromise(); + auto future = promise.GetFuture(); + + ThreadPool.SafeAddFunc( + [action = std::move(Actions[action]), promise = std::move(promise)] () mutable { + try { + action(); + promise.SetValue(); + } catch (...) { + Y_ABORT("should not happen: %s", CurrentExceptionMessage().c_str()); + } + } + ); + + return future; +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/executor.h b/cloud/filestore/tools/testing/replay/lib/executor.h new file mode 100644 index 0000000000..8cd64c41e7 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/executor.h @@ -0,0 +1,40 @@ +#pragma once + +#include "public.h" + +#include + +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +class TExecutor +{ + using TActionList = TVector>; + +private: + const TAppContext& Context; + const TActionList Actions; + + TAdaptiveLock Lock; + TAdaptiveThreadPool ThreadPool; + TManualEvent ReadyEvent; + ui32 DoneCount = 0; + +public: + TExecutor(const TAppContext& ctx, TActionList actions); + + void Run(); + +private: + NThreading::TFuture RunAction(ui32 action); +}; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/public.h b/cloud/filestore/tools/testing/replay/lib/public.h new file mode 100644 index 0000000000..62348a6e68 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/public.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct TAppContext; + +struct IClientFactory; +using IClientFactoryPtr = std::shared_ptr; + +struct IRequestGenerator; +using IRequestGeneratorPtr = std::shared_ptr; + +struct ITest; +using ITestPtr = std::shared_ptr; + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request.h b/cloud/filestore/tools/testing/replay/lib/request.h new file mode 100644 index 0000000000..a2ad107e58 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/request.h @@ -0,0 +1,61 @@ +#pragma once + +#include "public.h" + +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct TCompletedRequest +{ + NProto::EAction Action{}; + TDuration Elapsed; + NProto::TError Error; + bool Stop{}; + TCompletedRequest() = default; + + TCompletedRequest( + NProto::EAction action, + TInstant start, + NProto::TError error) noexcept + : Action(action) + , Elapsed(TInstant::Now() - start) + , Error(std::move(error)) + {} + TCompletedRequest(bool stop) noexcept + : Stop{stop} + {} +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct IRequestGenerator +{ + virtual ~IRequestGenerator() = default; + + virtual bool HasNextRequest() = 0; + // virtual TInstant NextRequestAt() = 0; + virtual NThreading::TFuture ExecuteNextRequest() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +IRequestGeneratorPtr CreateReplayRequestGenerator( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers); + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request_data.cpp b/cloud/filestore/tools/testing/replay/lib/request_data.cpp new file mode 100644 index 0000000000..2a8247b102 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/request_data.cpp @@ -0,0 +1,639 @@ +#include "request.h" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +const ui64 SEGMENT_SIZE = 128; + +struct TSegment +{ + ui64 Handle = 0; + ui64 LastWriteRequestId = 0; +}; + +static_assert(sizeof(TSegment) <= SEGMENT_SIZE); + +bool Compare( + const TSegment& expected, + const TSegment& actual, + TString* message) +{ + TStringBuilder sb; + + if (expected.Handle != actual.Handle) { + sb << "expected.Handle != actual.Handle: " + << expected.Handle << " != " << actual.Handle; + } + + if (expected.LastWriteRequestId != actual.LastWriteRequestId) { + if (sb.Size()) { + sb << ", "; + } + + sb << "expected.LastWriteRequestId != actual.LastWriteRequestId: " + << expected.LastWriteRequestId + << " != " << actual.LastWriteRequestId; + } + + *message = sb; + + return sb.Empty(); +} + +struct TSegments: TVector, TAtomicRefCount +{ +}; + +using TSegmentsPtr = TIntrusivePtr; + +//////////////////////////////////////////////////////////////////////////////// + +struct THandleInfo +{ + TString Name; + ui64 Handle = 0; + ui64 Size = 0; + TSegmentsPtr Segments; + ui64 LastSlot = 0; + + THandleInfo() = default; + + THandleInfo( + TString name, + ui64 handle, + ui64 size, + TSegmentsPtr segments) noexcept + : Name(std::move(name)) + , Handle(handle) + , Size(size) + , Segments(std::move(segments)) + {} +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TDataRequestGenerator final + : public IRequestGenerator + , public std::enable_shared_from_this +{ +private: + static constexpr ui32 DefaultBlockSize = 4_KB; + + const NProto::TDataLoadSpec Spec; + const TString FileSystemId; + const NProto::THeaders Headers; + + TLog Log; + + ISessionPtr Session; + + TVector> Actions; + ui64 TotalRate = 0; + + TMutex StateLock; + ui64 LastWriteRequestId = 0; + + TVector IncompleteHandleInfos; + TVector HandleInfos; + + ui64 ReadBytes = DefaultBlockSize; + ui64 WriteBytes = DefaultBlockSize; + double AppendProbability = 1; + ui64 BlockSize = DefaultBlockSize; + ui64 InitialFileSize = 0; + + std::atomic LastRequestId = 0; + +public: + TDataRequestGenerator( + NProto::TDataLoadSpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) + { + Log = logging->CreateLog(Headers.GetClientId()); + + if (auto size = Spec.GetBlockSize()) { + BlockSize = size; + Y_ENSURE(BlockSize % DefaultBlockSize == 0); + } + if (auto bytes = Spec.GetReadBytes()) { + ReadBytes = bytes; + } + if (auto bytes = Spec.GetWriteBytes()) { + WriteBytes = bytes; + } + if (auto p = Spec.GetAppendPercentage()) { + AppendProbability = p / 100.; + } + + InitialFileSize = Spec.GetInitialFileSize(); + + if (Spec.GetValidationEnabled()) { + Y_ENSURE( + InitialFileSize % SEGMENT_SIZE == 0, + Sprintf( + "InitialFileSize (%lu) %% SEGMENT_SIZE (%lu) != 0", + InitialFileSize, + SEGMENT_SIZE + ) + ); + + Y_ENSURE( + WriteBytes % SEGMENT_SIZE == 0, + Sprintf( + "WriteBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", + WriteBytes, + SEGMENT_SIZE + ) + ); + + Y_ENSURE( + ReadBytes % SEGMENT_SIZE == 0, + Sprintf( + "ReadBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", + ReadBytes, + SEGMENT_SIZE + ) + ); + } + + for (const auto& action: Spec.GetActions()) { + Y_ENSURE(action.GetRate() > 0, "please specify positive action rate"); + + TotalRate += action.GetRate(); + Actions.emplace_back(std::make_pair(TotalRate, action.GetAction())); + } + + Y_ENSURE(!Actions.empty(), "please specify at least one action for the test spec"); + } + + bool HasNextRequest() override + { + return true; + } + + TInstant NextRequestAt() override + { + return TInstant::Max(); + } + + NThreading::TFuture ExecuteNextRequest() override + { + const auto& action = PeekNextAction(); + switch (action) { + case NProto::ACTION_READ: + return DoRead(); + case NProto::ACTION_WRITE: + return DoWrite(); + default: + Y_ABORT("unexpected action: %u", (ui32)action); + } + } + +private: + NProto::EAction PeekNextAction() + { + auto number = RandomNumber(TotalRate); + auto it = LowerBound( + Actions.begin(), + Actions.end(), + number, + [] (const auto& pair, ui64 b) { return pair.first < b; }); + + Y_ABORT_UNLESS(it != Actions.end()); + return it->second; + } + + TFuture DoCreateHandle() + { + static const int flags = ProtoFlag(NProto::TCreateHandleRequest::E_CREATE) + | ProtoFlag(NProto::TCreateHandleRequest::E_EXCLUSIVE) + | ProtoFlag(NProto::TCreateHandleRequest::E_READ) + | ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); + + auto started = TInstant::Now(); + TGuard guard(StateLock); + auto name = GenerateNodeName(); + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + request->SetFlags(flags); + + auto self = weak_from_this(); + return Session->CreateHandle(CreateCallContext(), std::move(request)).Apply( + [=, name = std::move(name)] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleCreateHandle(future, name, started); + } + + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "cancelled")}); + }); + } + + TFuture HandleCreateHandle( + const TFuture& future, + const TString& name, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + auto handle = response.GetHandle(); + auto& infos = InitialFileSize ? IncompleteHandleInfos : HandleInfos; + with_lock (StateLock) { + TSegmentsPtr segments; + + if (Spec.GetValidationEnabled()) { + segments = new TSegments(); + segments->resize(Spec.GetInitialFileSize() / SEGMENT_SIZE); + } + + infos.emplace_back(name, handle, 0, std::move(segments)); + } + + NThreading::TFuture setAttr; + if (InitialFileSize) { + static const int flags = + ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetNodeId(response.GetNodeAttr().GetId()); + request->SetFlags(flags); + request->MutableUpdate()->SetSize(InitialFileSize); + + setAttr = Session->SetNodeAttr(CreateCallContext(), std::move(request)); + } else { + setAttr = NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); + } + + return setAttr.Apply( + [=] (const TFuture& f) { + return HandleResizeAfterCreateHandle(f, name, started); + }); + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("create handle for %s has failed: %s", + name.Quote().c_str(), + FormatError(error).c_str()); + + return NThreading::MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + error + }); + } + } + + TCompletedRequest HandleResizeAfterCreateHandle( + const TFuture& future, + const TString& name, + TInstant started) + { + try { + const auto& response = future.GetValue(); + CheckResponse(response); + bool handleFound = false; + with_lock (StateLock) { + ui32 handleIdx = 0; + while (handleIdx < IncompleteHandleInfos.size()) { + auto& hinfo = IncompleteHandleInfos[handleIdx]; + if (hinfo.Name == name) { + handleFound = true; + + hinfo.Size = Max( + hinfo.Size, + response.GetNode().GetSize()); + + STORAGE_INFO( + "updated file size for handle %lu and file %s" + ", size=%lu", + hinfo.Handle, + name.Quote().c_str(), + hinfo.Size); + + break; + } + + ++handleIdx; + } + + if (handleIdx < IncompleteHandleInfos.size()) { + DoSwap( + IncompleteHandleInfos[handleIdx], + IncompleteHandleInfos.back()); + + HandleInfos.push_back( + std::move(IncompleteHandleInfos.back())); + + IncompleteHandleInfos.pop_back(); + } + } + + if (!handleFound) { + STORAGE_WARN("handle for file %s not found", + name.Quote().c_str()); + } + + return { + NProto::ACTION_CREATE_HANDLE, + started, + response.GetError() + }; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("set node attr handle for %s has failed: %s", + name.Quote().c_str(), + FormatError(error).c_str()); + + return { + NProto::ACTION_CREATE_HANDLE, + started, + error + }; + } + } + + TFuture DoRead() + { + TGuard guard(StateLock); + if (HandleInfos.empty()) { + return DoCreateHandle(); + } + + auto handleInfo = GetHandleInfo(); + if (handleInfo.Size < ReadBytes) { + return DoWrite(handleInfo); + } + + const auto started = TInstant::Now(); + const ui64 slotOffset = PickSlot(handleInfo, ReadBytes); + const ui64 byteOffset = slotOffset * ReadBytes; + + auto request = CreateRequest(); + request->SetHandle(handleInfo.Handle); + request->SetOffset(byteOffset); + request->SetLength(ReadBytes); + + auto self = weak_from_this(); + return Session->ReadData(CreateCallContext(), std::move(request)).Apply( + [=] (const TFuture& future){ + if (auto ptr = self.lock()) { + return ptr->HandleRead( + future, + handleInfo, + started, + byteOffset + ); + } + + return TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_FAIL, "cancelled")}; + }); + } + + TCompletedRequest HandleRead( + const TFuture& future, + THandleInfo handleInfo, + TInstant started, + ui64 byteOffset) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + const auto& buffer = response.GetBuffer(); + + ui64 segmentId = byteOffset / SEGMENT_SIZE; + for (ui64 offset = 0; offset < ReadBytes; offset += SEGMENT_SIZE) { + const TSegment* segment = + reinterpret_cast(buffer.data() + offset); + if (Spec.GetValidationEnabled()) { + Y_ABORT_UNLESS(handleInfo.Segments); + + auto& segments = *handleInfo.Segments; + Y_ABORT_UNLESS(segmentId < segments.size()); + + TString message; + if (!Compare(segments[segmentId], *segment, &message)) { + throw TServiceError(E_FAIL) + << Sprintf("Validation failed: %s", message.c_str()); + } + } + + ++segmentId; + } + + with_lock (StateLock) { + HandleInfos.emplace_back(std::move(handleInfo)); + } + + return {NProto::ACTION_READ, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("read for %s has failed: %s", + handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_READ, started, error}; + } + } + + TFuture DoWrite(THandleInfo handleInfo = {}) + { + TGuard guard(StateLock); + if (!handleInfo.Handle) { + if (HandleInfos.empty()) { + return DoCreateHandle(); + } + + handleInfo = GetHandleInfo(); + } + + const auto started = TInstant::Now(); + ui64 byteOffset = handleInfo.Size; + if (RandomNumber() < AppendProbability) { + handleInfo.Size += WriteBytes; + } else { + handleInfo.Size = Max(handleInfo.Size, WriteBytes); + const ui64 slotOffset = PickSlot(handleInfo, WriteBytes); + byteOffset = slotOffset * WriteBytes; + } + + auto request = CreateRequest(); + request->SetHandle(handleInfo.Handle); + request->SetOffset(byteOffset); + + ++LastWriteRequestId; + + TString buffer(WriteBytes, '\0'); + ui64 segmentId = byteOffset / SEGMENT_SIZE; + for (ui64 offset = 0; offset < WriteBytes; offset += SEGMENT_SIZE) { + TSegment* segment = + reinterpret_cast(buffer.begin() + offset); + segment->Handle = handleInfo.Handle; + segment->LastWriteRequestId = LastWriteRequestId; + + if (Spec.GetValidationEnabled()) { + Y_ABORT_UNLESS(handleInfo.Segments); + + auto& segments = *handleInfo.Segments; + if (segments.size() <= segmentId) { + segments.emplace_back(); + } + + segments[segmentId] = *segment; + } + + ++segmentId; + } + *request->MutableBuffer() = std::move(buffer); + + auto self = weak_from_this(); + return Session->WriteData(CreateCallContext(), std::move(request)).Apply( + [=] (const TFuture& future){ + if (auto ptr = self.lock()) { + return ptr->HandleWrite(future, handleInfo, started); + } + + return TCompletedRequest{ + NProto::ACTION_WRITE, + started, + MakeError(E_FAIL, "cancelled")}; + }); + + } + + TCompletedRequest HandleWrite( + const TFuture& future, + THandleInfo handleInfo, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + with_lock (StateLock) { + HandleInfos.emplace_back(std::move(handleInfo)); + } + + return {NProto::ACTION_WRITE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("write on %s has failed: %s", + handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_WRITE, started, error}; + } + } + + template + std::shared_ptr CreateRequest() + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->CopyFrom(Headers); + + return request; + } + + TString GenerateNodeName() + { + return TStringBuilder() << Headers.GetClientId() << ":" << CreateGuidAsString(); + } + + THandleInfo GetHandleInfo() + { + Y_ABORT_UNLESS(!HandleInfos.empty()); + ui64 index = RandomNumber(HandleInfos.size()); + std::swap(HandleInfos[index], HandleInfos.back()); + + auto handle = HandleInfos.back(); + HandleInfos.pop_back(); + + return handle; + } + + template + void CheckResponse(const T& response) + { + if (HasError(response)) { + throw TServiceError(response.GetError()); + } + } + + TIntrusivePtr CreateCallContext() + { + return MakeIntrusive( + LastRequestId.fetch_add(1, std::memory_order_relaxed)); + } + + ui64 PickSlot(THandleInfo& handleInfo, ui64 reqBytes) + { + const ui64 slotCount = handleInfo.Size / reqBytes; + Y_ABORT_UNLESS(slotCount); + if (Spec.GetSequential()) { + return handleInfo.LastSlot = (handleInfo.LastSlot + 1) % slotCount; + } + + return RandomNumber(slotCount); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IRequestGeneratorPtr CreateDataRequestGenerator( + NProto::TDataLoadSpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) +{ + return std::make_shared( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)); +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request_index.cpp b/cloud/filestore/tools/testing/replay/lib/request_index.cpp new file mode 100644 index 0000000000..f6bf0cccb5 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/request_index.cpp @@ -0,0 +1,684 @@ +#include "request.h" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TIndexRequestGenerator final + : public IRequestGenerator + , public std::enable_shared_from_this +{ +private: + static constexpr ui32 LockLength = 4096; + + struct TNode + { + TString Name; + NProto::TNodeAttr Attrs; + }; + + struct THandle + { + TString Path; + ui64 Handle = 0; + }; + +private: + const NProto::TIndexLoadSpec Spec; + const TString FileSystemId; + const NProto::THeaders Headers; + const ui64 OwnerId = RandomNumber(100500u); + + TLog Log; + + ISessionPtr Session; + + TVector> Actions; + ui64 TotalRate = 0; + + TMutex StateLock; + THashMap StagedNodes; + THashMap Nodes; + + THashMap Handles; + + THashSet Locks; + THashSet StagedLocks; + + ui64 LastRequestId = 0; + +public: + TIndexRequestGenerator( + NProto::TIndexLoadSpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) + { + Log = logging->CreateLog(Headers.GetClientId()); + + for (const auto& action: Spec.GetActions()) { + Y_ENSURE(action.GetRate() > 0, "please specify positive action rate"); + + TotalRate += action.GetRate(); + Actions.emplace_back(std::make_pair(TotalRate, action.GetAction())); + } + + Y_ENSURE(!Actions.empty(), "please specify at least one action for the test spec"); + } + + bool HasNextRequest() override + { + return true; + } + + TInstant NextRequestAt() override + { + return TInstant::Max(); + } + + NThreading::TFuture ExecuteNextRequest() override + { + const auto& action = PeekNextAction(); + switch (action) { + case NProto::ACTION_CREATE_NODE: + return DoCreateNode(); + case NProto::ACTION_RENAME_NODE: + return DoRenameNode(); + case NProto::ACTION_REMOVE_NODE: + return DoUnlinkNode(); + case NProto::ACTION_CREATE_HANDLE: + return DoCreateHandle(); + case NProto::ACTION_DESTROY_HANDLE: + return DoDestroyHandle(); + case NProto::ACTION_GET_NODE_ATTR: + return DoGetNodeAttr(); + case NProto::ACTION_ACQUIRE_LOCK: + return DoAcquireLock(); + case NProto::ACTION_RELEASE_LOCK: + return DoReleaseLock(); + default: + Y_ABORT("unexpected action: %u", (ui32)action); + } + } + +private: + NProto::EAction PeekNextAction() + { + auto number = RandomNumber(TotalRate); + auto it = LowerBound( + Actions.begin(), + Actions.end(), + number, + [] (const auto& pair, ui64 b) { return pair.first < b; }); + + Y_ABORT_UNLESS(it != Actions.end()); + return it->second; + } + + TFuture DoCreateNode() + { + TGuard guard(StateLock); + + auto name = GenerateNodeName(); + StagedNodes[name] = {}; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + request->MutableFile()->SetMode(0777); + + auto started = TInstant::Now(); + auto self = weak_from_this(); + return Session->CreateNode(CreateCallContext(), std::move(request)).Apply( + [=] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleCreateNode(future, name, started); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleCreateNode( + const TFuture& future, + const TString& name, + TInstant started) + { + TGuard guard(StateLock); + Y_ABORT_UNLESS(StagedNodes.erase(name)); + + try { + auto response = future.GetValue(); + CheckResponse(response); + + Nodes[name] = TNode{name, response.GetNode()}; + return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("create node %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_NODE, started, error}; + } + } + + TFuture DoRenameNode() + { + TGuard guard(StateLock); + + if (Nodes.empty()) { + return DoCreateNode(); + } + + auto started = TInstant::Now(); + + auto it = Nodes.begin(); + auto old = it->first; + auto newbie = GenerateNodeName(); + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(old); + request->SetNewParentId(RootNodeId); + request->SetNewName(newbie); + + StagedNodes[newbie] = {}; + StagedNodes[old] = std::move(it->second); + Nodes.erase(it); + + auto self = weak_from_this(); + return Session->RenameNode(CreateCallContext(), std::move(request)).Apply( + [=, old = std::move(old), newbie = std::move(newbie)] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleRenameNode(future, old, newbie, started); + } + + return TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleRenameNode( + const TFuture& future, + const TString& old, + const TString& newbie, + TInstant started) + { + TGuard guard(StateLock); + + auto node = std::move(StagedNodes[old]); + StagedNodes.erase(old); + StagedNodes.erase(newbie); + + try { + auto response = future.GetValue(); + CheckResponse(response); + + Nodes[newbie] = std::move(node); + return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("rename node %s has failed: %s", + old.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_RENAME_NODE, started, error}; + } + } + + TFuture DoUnlinkNode() + { + TGuard guard(StateLock); + if (Nodes.empty()) { + return DoCreateNode(); + } + + auto started = TInstant::Now(); + + auto it = Nodes.begin(); + auto name = it->first; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + + StagedNodes[name] = std::move(it->second); + Nodes.erase(it); + + auto self = weak_from_this(); + return Session->UnlinkNode(CreateCallContext(), std::move(request)).Apply( + [=, name = std::move(name)] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleUnlinkNode(future, name, started); + } + + return TCompletedRequest{ + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleUnlinkNode( + const TFuture& future, + const TString& name, + TInstant started) + { + with_lock (StateLock) { + StagedNodes.erase(name); + } + + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("unlink for %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_REMOVE_NODE, started, error}; + } + } + + TFuture DoCreateHandle() + { + static const int flags = ProtoFlag(NProto::TCreateHandleRequest::E_READ) + | ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); + + TGuard guard(StateLock); + if (Nodes.empty()) { + return DoCreateNode(); + } + + auto started = TInstant::Now(); + + auto it = Nodes.begin(); + if (Nodes.size() > 1) { + std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); + } + + auto name = it->first; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + request->SetFlags(flags); + + auto self = weak_from_this(); + return Session->CreateHandle(CreateCallContext(), std::move(request)).Apply( + [=, name = std::move(name)] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleCreateHandle(future, name, started); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleCreateHandle( + const TFuture& future, + const TString name, + TInstant started) + { + TGuard guard(StateLock); + + try { + auto response = future.GetValue(); + CheckResponse(response); + + auto handle = response.GetHandle(); + Handles[handle] = THandle{name, handle}; + + return {NProto::ACTION_CREATE_HANDLE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("create handle for %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_HANDLE, started, error}; + } + } + + TFuture DoDestroyHandle() + { + TGuard guard(StateLock); + if (Handles.empty()) { + return DoCreateHandle(); + } + + auto started = TInstant::Now(); + auto it = Handles.begin(); + + ui64 handle = it->first; + auto name = it->second.Path; + Handles.erase(it); + if (auto it = Locks.find(handle); it != Locks.end()) { + Locks.erase(it); + } + if (auto it = StagedLocks.find(handle); it != StagedLocks.end()) { + StagedLocks.erase(it); + } + + auto request = CreateRequest(); + request->SetHandle(handle); + + auto self = weak_from_this(); + return Session->DestroyHandle(CreateCallContext(), std::move(request)).Apply( + [=, name = std::move(name)] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleDestroyHandle(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_DESTROY_HANDLE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TFuture DoGetNodeAttr() + { + TGuard guard(StateLock); + if (Nodes.empty()) { + return DoCreateNode(); + } + + auto started = TInstant::Now(); + auto it = Nodes.begin(); + if (Nodes.size() > 1) { + std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); + } + + auto name = it->first; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + + auto self = weak_from_this(); + STORAGE_DEBUG("GetNodeAttr client started"); + return Session->GetNodeAttr(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleGetNodeAttr(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleGetNodeAttr( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + STORAGE_DEBUG("GetNodeAttr client completed"); + CheckResponse(response); + TGuard guard(StateLock); + if (response.GetNode().SerializeAsString() != + Nodes[name].Attrs.SerializeAsString()) + { + auto error = MakeError( + E_FAIL, + TStringBuilder() + << "attributes are not equal for node " << name << ": " + << response.GetNode().DebugString().Quote() + << " != " << Nodes[name].Attrs.DebugString().Quote()); + STORAGE_ERROR(error.GetMessage().c_str()); + } + return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "get node attr %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_GET_NODE_ATTR, started, error}; + } + } + + TCompletedRequest HandleDestroyHandle( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_DESTROY_HANDLE, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("destroy handle %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_DESTROY_HANDLE, started, error}; + } + } + + TFuture DoAcquireLock() + { + TGuard guard(StateLock); + if (Handles.empty()) { + return DoCreateHandle(); + } + + auto started = TInstant::Now(); + auto it = Handles.begin(); + while (it != Handles.end() && (Locks.contains(it->first) || StagedLocks.contains(it->first))) { + ++it; + } + + if (it == Handles.end()) { + return DoCreateHandle(); + } + + auto handle = it->first; + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto self = weak_from_this(); + return Session->AcquireLock(CreateCallContext(), std::move(request)).Apply( + [=] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleAcquireLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleAcquireLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it == StagedLocks.end()) { + // nothing todo, file was removed + Y_ABORT_UNLESS(!Locks.contains(handle)); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } + + StagedLocks.erase(it); + + try { + const auto& response = future.GetValue(); + CheckResponse(response); + + Locks.insert(handle); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("acquire lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_ACQUIRE_LOCK, started, error}; + } + } + + TFuture DoReleaseLock() + { + TGuard guard(StateLock); + if (Locks.empty()) { + return DoAcquireLock(); + } + + auto it = Locks.begin(); + auto handle = *it; + + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + Locks.erase(it); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto started = TInstant::Now(); + auto self = weak_from_this(); + return Session->ReleaseLock(CreateCallContext(), std::move(request)).Apply( + [=] (const TFuture& future) { + if (auto ptr = self.lock()) { + return ptr->HandleReleaseLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleReleaseLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it != StagedLocks.end()) { + StagedLocks.erase(it); + } + + try { + CheckResponse(future.GetValue()); + return {NProto::ACTION_RELEASE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR("release lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_RELEASE_LOCK, started, error}; + } + } + + template + std::shared_ptr CreateRequest() + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->CopyFrom(Headers); + + return request; + } + + TString GenerateNodeName() + { + return TStringBuilder() << Headers.GetClientId() << ":" << CreateGuidAsString(); + } + + template + void CheckResponse(const T& response) + { + if (HasError(response)) { + throw TServiceError(response.GetError()); + } + } + + TIntrusivePtr CreateCallContext() + { + return MakeIntrusive(AtomicIncrement(LastRequestId)); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IRequestGeneratorPtr CreateIndexRequestGenerator( + NProto::TIndexLoadSpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) +{ + return std::make_shared( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)); +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay.cpp b/cloud/filestore/tools/testing/replay/lib/request_replay.cpp new file mode 100644 index 0000000000..c0379083ce --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/request_replay.cpp @@ -0,0 +1,2678 @@ +/* + +TODO: +create file/dir modes +create handle modes (now rw) +compare log and actual result ( S_OK E_FS_NOENT ...) + + + + + +*/ + +#include "request.h" + +#include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" +#include "cloud/filestore/tools/analytics/libs/event-log/dump.h" +// #include "dump.h" +#include "library/cpp/aio/aio.h" +#include "library/cpp/eventlog/iterator.h" +#include "library/cpp/testing/unittest/registar.h" +#include "util/folder/dirut.h" +#include "util/folder/path.h" +#include "util/system/fs.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +const ui64 SEGMENT_SIZE = 128; + +struct TSegment +{ + ui64 Handle = 0; + ui64 LastWriteRequestId = 0; +}; + +static_assert(sizeof(TSegment) <= SEGMENT_SIZE); +/* +bool Compare(const TSegment& expected, const TSegment& actual, TString* message) +{ + TStringBuilder sb; + + if (expected.Handle != actual.Handle) { + sb << "expected.Handle != actual.Handle: " << expected.Handle + << " != " << actual.Handle; + } + + if (expected.LastWriteRequestId != actual.LastWriteRequestId) { + if (sb.Size()) { + sb << ", "; + } + + sb << "expected.LastWriteRequestId != actual.LastWriteRequestId: " + << expected.LastWriteRequestId + << " != " << actual.LastWriteRequestId; + } + + *message = sb; + + return sb.Empty(); +} +*/ +struct TSegments + : TVector + , TAtomicRefCount +{ +}; + +using TSegmentsPtr = TIntrusivePtr; + +//////////////////////////////////////////////////////////////////////////////// + +struct THandleInfo +{ + TString Name; + ui64 Handle = 0; + ui64 Size = 0; + TSegmentsPtr Segments; + ui64 LastSlot = 0; + + THandleInfo() = default; + + THandleInfo( + TString name, + ui64 handle, + ui64 size, + TSegmentsPtr segments) noexcept + : Name(std::move(name)) + , Handle(handle) + , Size(size) + , Segments(std::move(segments)) + {} +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TReplayRequestGenerator final + : public IRequestGenerator + , public std::enable_shared_from_this +{ +private: + // static constexpr ui32 DefaultBlockSize = 4_KB; + + const NProto::TReplaySpec Spec; + // const + TString FileSystemId; + const NProto::THeaders Headers; + + TLog Log; + + ISessionPtr Session; + + TVector> Actions; + // ui64 TotalRate = 0; + + TMutex StateLock; + ui64 LastWriteRequestId = 0; + + using THandleLog = ui64; + using THandleLocal = ui64; + using TNodeLog = ui64; + using TNodeLocal = ui64; + + TVector IncompleteHandleInfos; + TVector HandleInfos; + THashMap HandleInfosh; + THashMap HandlesLogToActual; + + // ui64 ReadBytes = DefaultBlockSize; + // ui64 WriteBytes = DefaultBlockSize; + // double AppendProbability = 1; + // ui64 BlockSize = DefaultBlockSize; + ui64 InitialFileSize = 0; + + std::atomic LastRequestId = 0; + + // NEventLog::IIterator * + THolder EventlogIterator; + TConstEventPtr EventPtr; + int EventMessageNumber = 0; + NProto::TProfileLogRecord* messagep{}; + // TString FileSystemId; + + static constexpr ui32 LockLength = 4096; + + const ui64 OwnerId = RandomNumber(100500u); + + struct TNode + { + TString Name; + NProto::TNodeAttr Attrs; + + TNodeLog ParentLog = 0; + }; + + struct THandle + { + TString Path; + ui64 Handle = 0; + }; + + THashMap StagedNodes; + THashMap Nodes; + // THashMap NodesLogToActual; + THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; + THashMap NodePath{{RootNodeId, "/"}}; + + //THashMap FileHandles; + + THashMap Handles; + + THashSet Locks; + THashSet StagedLocks; + + NAsyncIO::TAsyncIOService AsyncIO; + // TString ReplayRoot; + ui64 TimestampMcs{}; + // ui64 DurationMcs{}; + TInstant Started; + // THashMap OpenFiles; + // THashMap OpenHandles; + THashMap OpenHandles; + THashMap KnownLogNodes; + +public: + TReplayRequestGenerator( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) + { + Log = logging->CreateLog(Headers.GetClientId()); + + // DUMP(spec); + + // ReplayRoot = spec.GetReplayRoot(); + + AsyncIO.Start(); + + /* + if (auto size = Spec.GetBlockSize()) { + BlockSize = size; + Y_ENSURE(BlockSize % DefaultBlockSize == 0); + } + if (auto bytes = Spec.GetReadBytes()) { + ReadBytes = bytes; + } + if (auto bytes = Spec.GetWriteBytes()) { + WriteBytes = bytes; + } + if (auto p = Spec.GetAppendPercentage()) { + AppendProbability = p / 100.; + } + + InitialFileSize = Spec.GetInitialFileSize(); + + if (Spec.GetValidationEnabled()) { + Y_ENSURE( + InitialFileSize % SEGMENT_SIZE == 0, + Sprintf( + "InitialFileSize (%lu) %% SEGMENT_SIZE (%lu) != 0", + InitialFileSize, + SEGMENT_SIZE + ) + ); + + Y_ENSURE( + WriteBytes % SEGMENT_SIZE == 0, + Sprintf( + "WriteBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", + WriteBytes, + SEGMENT_SIZE + ) + ); + + Y_ENSURE( + ReadBytes % SEGMENT_SIZE == 0, + Sprintf( + "ReadBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", + ReadBytes, + SEGMENT_SIZE + ) + ); + } + */ + + /* + for (const auto& action: Spec.GetActions()) { + Y_ENSURE(action.GetRate() > 0, "please specify positive + action rate"); + + TotalRate += action.GetRate(); + Actions.emplace_back(std::make_pair(TotalRate, + action.GetAction())); + } + + Y_ENSURE(!Actions.empty(), "please specify at least one action + for the test spec"); + */ + + NEventLog::TOptions options; + // DUMP(Spec); + // Cerr << "SPEC=" << Spec; + options.FileName = Spec.GetFileName(); + options.SetForceStrongOrdering(true); + // options.ForceStreamMode + // THolder it = CreateIterator(options, fac); + // DUMP(options.FileName); + // THolder it + EventlogIterator = CreateIterator(options); + // EventPtr = EventlogIterator->Next(); + // DUMP("opened"); + // NEventLog::IIterator* i + // EventlogIterator = it.Get(); + // DUMP((long)i); + if (!Spec.GetReplayRoot().empty()) { + // auto fspath = + TFsPath(Spec.GetReplayRoot()).MkDirs(); + } + } + + ~TReplayRequestGenerator() + { + AsyncIO.Stop(); + } + + TNodeLocal NodeIdMapped(const TNodeLog id) + { + if (const auto it = NodesLogToLocal.find(id); + it != NodesLogToLocal.end()) + { + return it->second; + } + + STORAGE_INFO( + "node not found " << id << " map size=" << NodesLogToLocal.size()); + // DUMP("but known=", KnownLogNodes.contains(id)); + return 0; + } + + THandleLocal HandleIdMapped(const THandleLog id) + { + if (const auto it = HandlesLogToActual.find(id); + it != HandlesLogToActual.end()) + { + return it->second; + } + STORAGE_INFO( + "handle not found " << id + << " map size=" << HandlesLogToActual.size()); + return 0; + } + + bool UseFs() + { + return !Spec.GetReplayRoot().empty(); + } + + void Advance() + { + EventPtr = EventlogIterator->Next(); + // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); + if (!EventPtr) { + return; + } + // if (EventPtr) { + // DUMP(EventPtr->GetName(),EventPtr->Class,EventPtr->FrameId); + //,EventPtr->ToString() + + // const NProto::TProfileLogRecord* + messagep = const_cast( + dynamic_cast( + EventPtr->GetProto())); + if (!messagep) { + // DUMP("nomesssss"); + return; + } + + // DUMP(messagep->GetClassData(), messagep->GetFileSystemId()); + if (FileSystemId.empty()) { + FileSystemId = TString{messagep->GetFileSystemId()}; + } + + EventMessageNumber = messagep->GetRequests().size(); + } + + bool HasNextRequest() override + { + // DUMP("hasnext?", !!EventPtr); + if (!EventPtr) { + // EventPtr = EventlogIterator->Next(); + Advance(); + } + // DUMP("hasnext=", !!EventPtr); + return !!EventPtr; + } + + /* TInstant NextRequestAt() override + { + return TInstant::Max(); + } + */ + + NThreading::TFuture ExecuteNextRequest() override + { + // DUMP("en"); + // auto *ev = EventlogIterator.Get(); + // EventPtr = EventlogIterator.Get(); +#if 0 + if (0) { + while (auto ev = EventlogIterator->Next()) { + // auto ev = EventlogIterator->Next(); + + DUMP(ev->GetName(), ev->ToString()); + + // event->Class; + + const auto* message = + dynamic_cast( + ev->GetProto()); + if (!message) { + DUMP("nomess"); + continue; + } + // const auto *message = ev->GetProto(); + // DUMP(message->AsJSON()); + // DUMP(message->GetTypeName()); + for (const auto& r: message->GetRequests()) { + DUMP(r.GetTypeName(), r.GetBlobsInfo(), r.GetRanges()); + DUMP(RequestName(r.GetRequestType())); + // TBasicString >&& () noexcept + // RequestName(r.GetRequestType()) = "GetNodeAttr"; + // {"TimestampMcs":1725548818281695,"DurationMcs":128,"RequestType":33,"ErrorCode":0,"NodeInfo":{"ParentNodeId":2395,"NodeName":"pre-receive.sample","Flags":0,"Mode":509,"NodeId":2402,"Handle":0,"Size":544}} + // DUMP(r.AsJSON()); + // Cerr << r.AsJSON() << "\n"; + DUMP(r.GetMetadata()); + } + } + } +#endif + { + if (!HasNextRequest()) { + // DUMP("noreq"); + return MakeFuture(TCompletedRequest{}); + }; + // if (!EventPtr) {EventPtr = EventlogIterator->Next();} + + // NProto::TProfileLogRequestInfo request; + + // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); + // do { + for (; EventPtr; + + // EventMessageNumber = 0, + // EventPtr = EventlogIterator->Next() + Advance()) + { + /* + if (!EventPtr) { + continue; + } + */ + // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); + // if (EventPtr) { + /* + const NProto::TProfileLogRecord* messagep = + dynamic_cast( EventPtr->GetProto()); + */ + if (!messagep) { + // DUMP("nomess"); + continue; + } + + /* const auto i = message->GetRequests().begin(); + for (const auto& r: message->GetRequests()) { + r.GetRequestType(); + } + */ + STORAGE_DEBUG("Processing %d", EventMessageNumber); + // DUMP(EventMessageNumber, messagep->GetRequests().size()); + /* + if (EventMessageNumber >= + messagep->GetRequests().size()) { DUMP("nonextreq"); + continue; + } + for (; EventMessageNumber < + messagep->GetRequests().size();) { auto request = + messagep->GetRequests()[EventMessageNumber++]; + */ + for (; EventMessageNumber > 0;) { + auto request = + messagep->GetRequests()[--EventMessageNumber]; + // TODO check ranges + // DUMP(request.AsJSON()); + // Cerr << request.AsJSON() << "\n"; + STORAGE_DEBUG("message json=" << request.AsJSON()); + // "TimestampMcs":1725981613498599,"DurationMcs":2101, + // TODO WTF + // 2024-09-10T14:52:07.145177Z nfs ListNodes + // 0.000385s S_OK {node_id=1, size=0} + // 2024-09-10T14:52:07.146009Z nfs ListNodes + // 0.000128s S_OK {node_id=1, size=0} + // 1970-01-01T00:14:52.767000Z nfs PingSession + // 0.000328s S_OK {no_info} + // 1970-01-01T00:14:53.768000Z nfs PingSession + // 0.000279s S_OK {no_info} + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + // DUMP(request.GetTimestampMcs(),TimestampMcs,timediff); + TimestampMcs = request.GetTimestampMcs(); + + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = TDuration::MicroSeconds( + timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); + + Sleep(slp); // TODO: calculate correct timescale here + } + // DUMP(current, Started, diff.MicroSeconds()); + Started = current; + + // DurationMcs = + // request.GetDurationMcs(); + + STORAGE_DEBUG( + "Processing message " + << EventMessageNumber + << " typename=" << request.GetTypeName() + << " type=" << request.GetRequestType() << " " + << RequestName(request.GetRequestType())); + + { + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest:: + ReadData: // NProto::ACTION_READ: + return DoReadData(request); + case EFileStoreRequest:: + WriteData: // NProto::ACTION_WRITE: + return DoWrite(request); + // static_cast(requestType) + case EFileStoreRequest:: + CreateNode: // NProto::ACTION_CREATE_NODE: + return DoCreateNode(request); + case EFileStoreRequest:: + RenameNode: // NProto::ACTION_RENAME_NODE: + return DoRenameNode(request); + case EFileStoreRequest:: + UnlinkNode: // NProto::ACTION_REMOVE_NODE: + return DoUnlinkNode(request); + case EFileStoreRequest:: + CreateHandle: // NProto::ACTION_CREATE_HANDLE: + return DoCreateHandle(request); + case EFileStoreRequest:: + DestroyHandle: // NProto::ACTION_DESTROY_HANDLE: + return DoDestroyHandle(request); + case EFileStoreRequest:: + GetNodeAttr: // NProto::ACTION_GET_NODE_ATTR: + return DoGetNodeAttr(request); + case EFileStoreRequest:: + AcquireLock: // NProto::ACTION_ACQUIRE_LOCK: + return DoAcquireLock(); + case EFileStoreRequest:: + ReleaseLock: // NProto::ACTION_RELEASE_LOCK: + return DoReleaseLock(); + + case EFileStoreRequest:: + AccessNode: // NProto::ACTION_CREATE_HANDLE: + return DoAccessNode(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + + continue; + + /* TODO: + 2024-09-19T23:54:21.789965Z :smoke DEBUG: + cloud/filestore/tools/testing/replay/lib/request_replay.cpp:475: + message + json={"TimestampMcs":1726503858215279,"DurationMcs":20865,"RequestType":10001,"ErrorCode":0,"BlobsInfo":[{"CommitId":4295169028,"Unique":1037938976620549,"Ranges":[{"NodeId":16064,"Offset":1179648,"Bytes":81920},{"NodeId":16065,"Offset":1179648,"Bytes":81920},{"NodeId":16066,"Offset":1179648,"Bytes":77824}]},{"CommitId":4295169028,"Unique":1636073302130950,"Ranges":[{"NodeId":16050,"Offset":0,"Bytes":262144},{"NodeId":16051,"Offset":0,"Bytes":16384},{"NodeId":16052,"Offset":0,"Bytes":49152},{"NodeId":16060,"Offset":0,"Bytes":20480},{"NodeId":16061,"Offset":0,"Bytes":32768}]},{"CommitId":4295169028,"Unique":1125899906843139,"Ranges":[{"NodeId":16050,"Offset":262144,"Bytes":262144}]},{"CommitId":4295169028,"Unique":211106232533764,"Ranges":[{"NodeId":16040,"Offset":0,"Bytes":49152}]},{"CommitId":4295169028,"Unique":1125899906843653,"Ranges":[{"NodeId":16050,"Offset":524288,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906843910,"Ranges":[{"NodeId":16050,"Offset":786432,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1952732650931715,"Ranges":[{"NodeId":16050,"Offset":1048576,"Bytes":262144},{"NodeId":16062,"Offset":1179648,"Bytes":114688},{"NodeId":16063,"Offset":1179648,"Bytes":77824}]},{"CommitId":4295169028,"Unique":1125899906844420,"Ranges":[{"NodeId":16050,"Offset":1310720,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906844677,"Ranges":[{"NodeId":16050,"Offset":1572864,"Bytes":262144}]},{"CommitId":4295169028,"Unique":52776558135558,"Ranges":[{"NodeId":16041,"Offset":1822720,"Bytes":12288}]},{"CommitId":4295169028,"Unique":1125899906845187,"Ranges":[{"NodeId":16050,"Offset":1835008,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845444,"Ranges":[{"NodeId":16041,"Offset":1835008,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845701,"Ranges":[{"NodeId":16050,"Offset":2097152,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845958,"Ranges":[{"NodeId":16041,"Offset":2097152,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906846211,"Ranges":[{"NodeId":16050,"Offset":2359296,"Bytes":262144}]},{"CommitId":4295169028,"Unique":439804651114244,"Ranges":[{"NodeId":16041,"Offset":2359296,"Bytes":102400}]},{"CommitId":4295169028,"Unique":1125899906846725,"Ranges":[{"NodeId":16050,"Offset":2621440,"Bytes":262144}]},{"CommitId":4295169028,"Unique":175921860448518,"Ranges":[{"NodeId":16050,"Offset":2883584,"Bytes":40960}]},{"CommitId":4295169028,"Unique":123145302315523,"Ranges":[{"NodeId":16067,"Offset":0,"Bytes":28672}]}]} + 2024-09-19T23:54:21.789970Z :smoke DEBUG: + cloud/filestore/tools/testing/replay/lib/request_replay.cpp:517: + Processing message 4294 + typename=NCloud.NFileStore.NProto.TProfileLogRequestInfo + type=10001 Flush + */ + + // listnodes ->. noeid -> createdir + + // 9: CreateSession + + default: + STORAGE_INFO( + "Uninmplemented action=" + << action << " " + << RequestName(request.GetRequestType())); + // Y_ABORT("unexpected action: %u", + // (ui32)action); + continue; + } + } + } + + // break; + + // message.get + } + STORAGE_DEBUG( + "Finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); + //} while (EventMessageNumber = 0, EventPtr = + // EventlogIterator->Next()); + + /* + // const auto& action = PeekNextAction(); + */ + /* + return MakeFuture(TCompletedRequest{ + NProto::EAction::ACTION_READ, + TInstant::Now(), + NProto::TError{}}); + // return DoRead(); + */ + + return MakeFuture(TCompletedRequest(true)); + } + } + +private: + /* + NProto::EAction PeekNextAction() + { + auto number = RandomNumber(TotalRate); + auto it = LowerBound( + Actions.begin(), + Actions.end(), + number, + [](const auto& pair, ui64 b) { return pair.first < b; }); + + Y_ABORT_UNLESS(it != Actions.end()); + return it->second; + } + */ + + TFuture DoAccessNode( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + if (Spec.GetNoRead()) { + return {}; + } + + // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} + TGuard guard(StateLock); + auto started = TInstant::Now(); + if (UseFs()) { + const auto node = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + + if (!node) { + STORAGE_ERROR( + "access fail: " << " no node=" + << r.GetNodeInfo().GetNodeId()); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_FAIL, "cancelled")}); + } + + auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); + int res = access(fname.c_str(), R_OK); + STORAGE_DEBUG( + "access " << node << " <- " << r.GetNodeInfo().GetNodeId() + << " = " << res); + return MakeFuture( + TCompletedRequest{NProto::ACTION_ACCESS_NODE, started, {}}); + } + return {}; + } + + // Recursive, no infinity loop check + TNodeLocal CreateDirIfMissingByNodeLog(TNodeLog nodeIdLog) + { + if (const auto& nodeIdLocal = NodeIdMapped(nodeIdLog)) { + // DUMP("Already created", nodeIdLog, "->", nodeIdLocal); + return nodeIdLocal; + } + + // DUMP(nodeIdLog); + + const auto& it = KnownLogNodes.find(nodeIdLog); + if (it == KnownLogNodes.end()) { + // DUMP("not known", NodePath, "but", NodeIdMapped(nodeIdLog)); + // maybe create in tmpdir + return 0; + } + /* + DUMP( + "WANACREEEEE", + nodeIdLog, + it->second.Name, + it->second.ParentLog, + NodeIdMapped(it->second.ParentLog)); + */ + auto parent = NodeIdMapped(it->second.ParentLog); + + if (!parent && it->second.ParentLog && + nodeIdLog != it->second.ParentLog) + { + // NodeIdMapped() + parent = CreateDirIfMissingByNodeLog(it->second.ParentLog); + // DUMP("create miss parent", it->second.ParentLog, " => ", parent); + } + + // if (!parent) + { + auto parentPath = PathByNode(NodeIdMapped(it->second.ParentLog)); + /* + DUMP( + "try1", + parentPath, + parent, + it->second.ParentLog, + parentPath, + NodeIdMapped(it->second.ParentLog)); + */ + // KnownLogNodes.contains() + if (parentPath.empty() && parent) { + parentPath = PathByNode(parent); + // DUMP("try2", parent, parentPath); + } + if (parentPath.empty()) { + parentPath = "/__lost__/"; + } + // DUMP("tryR", parent, it->second.ParentLog, parentPath); + + // const auto name = (parent ? PathByNode(parent) : "__lost__/") + + // ToString(logNodeId) + "/"; const auto name = parentPath + + // ToString(logNodeId) + "/"; + const auto name = parentPath + + (it->second.Name.empty() ? ToString(nodeIdLog) + : it->second.Name) + + "/"; + /* + DUMP( + "namegen", + parent, + PathByNode(parent), + it->second.Parent, + PathByNode(it->second.Parent), + parentPath, + logNodeId); + */ + const auto nodeId = Mkdir(Spec.GetReplayRoot() + name); + NodePath[nodeId] = name; + // DUMP("savepath", nodeId, name); + NodesLogToLocal[nodeIdLog] = nodeId; + + // DUMP("created lost", nodeid, name); + return nodeId; + } + /* + const auto name = PathByNode(parent) + "/" + it->second.Name; + const auto nodeid = Mkdir(Spec.GetReplayRoot() + "/" + name); + NodePath[nodeid] = name; + + return nodeid; + */ + } + + TFuture DoCreateHandle( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + // message + // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} + TGuard guard(StateLock); + auto started = TInstant::Now(); + if (UseFs()) { + /// DUMP("hhhhhhh================================================"); + TString relativePathName; + if (r.GetNodeInfo().GetNodeId()) { + if (auto path = PathByNode(r.GetNodeInfo().GetNodeId())) { + relativePathName = path; + } + // DUMP("open by nodeid", relativePathName); + // TODO: maybe open by fhandle + } + + if (relativePathName.empty()) { + auto parentNode = + NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + /* + DUMP( + "mapped", + parentNode, + r.GetNodeInfo().GetParentNodeId(), + PathByNode(parentNode)); + */ + if (!parentNode) { + parentNode = NodeIdMapped( + KnownLogNodes[r.GetNodeInfo().GetParentNodeId()] + .ParentLog); + // DUMP("known", parentNode); + } + + if (!parentNode && r.GetNodeInfo().GetParentNodeId() != + r.GetNodeInfo().GetNodeId()) + { + parentNode = CreateDirIfMissingByNodeLog( + r.GetNodeInfo().GetParentNodeId()); + // DUMP("create", parentNode); + } + + if (!parentNode) { + STORAGE_ERROR( + "create handle fail :" + << r.GetNodeInfo().GetHandle() + << " no parent=" << r.GetNodeInfo().GetParentNodeId()); + // DUMP( + // "noparent",r.GetNodeInfo().GetParentNodeId(),r.GetNodeInfo().GetHandle()); + // DUMP(NodesLogToActual); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "cancelled")}); + } + + auto nodename = r.GetNodeInfo().GetNodeName(); + if (nodename.empty() && r.GetNodeInfo().GetNodeId() != + r.GetNodeInfo().GetParentNodeId()) + { + nodename = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; + // DUMP("node name known", nodename); + } + auto parentpath = PathByNode(parentNode); + + if (nodename.empty() && + IsDir(Spec.GetReplayRoot() + parentpath)) + { + nodename = + KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name; + } + + // DUMP("can add + // name",KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name); + // DUMP(parentpath,nodename,NFs::Exists(Spec.GetReplayRoot() + + // parentpath)); + relativePathName = + //"/" + + // PathByNode(parentNode) + + // r.GetNodeInfo().GetNewNodeName(); + parentpath + nodename; + /* + DUMP( + "fname", + relativePathName, + parentNode, + parentpath, + PathByNode(parentNode), + nodename); + */ + } + // PathByNode(r.GetNodeInfo().GetNodeId())+ + // r.GetNodeInfo().GetNewNodeName(); + STORAGE_DEBUG( + "open " + << relativePathName // << " in " << parentNode << " <- " + << " handle=" << r.GetNodeInfo().GetHandle() + << " flags=" << r.GetNodeInfo().GetFlags() + << " mode=" << r.GetNodeInfo().GetMode() + << " node=" << r.GetNodeInfo().GetNodeId() + + ) + + ; + + // ui64 id = 0; + // DUMP("file", fname); + + /* + if (OpenFiles.contains(fname)) { + DUMP( + "already opened", + fname, + OpenFiles[fname], + r.GetNodeInfo().GetHandle()); + // return MakeFuture(TCompletedRequest{}); + } + */ + // DUMP(NFs::Exists(Spec.GetReplayRoot() + relativePathName)); + // TFileHandle + TFile FileHandle( + Spec.GetReplayRoot() + relativePathName, + // static_cast(r.GetNodeInfo().GetMode()) + OpenAlways | RdWr // RdOnly + + ); + + if (!FileHandle.IsOpen()) { + TFile test{ + Spec.GetReplayRoot() + relativePathName, + OpenAlways | RdWr}; + /* + DUMP( + " seconddtest", + test.IsOpen(), + test.GetName(), + test.GetLength(), + test.GetPosition(), + test.GetHandle()); + */ + } + + /* + DUMP( + FileHandle.IsOpen(), + FileHandle.GetPosition(), + FileHandle.GetLength()); + */ + + if (!FileHandle.IsOpen()) { + // DUMP("handle open fail", r.GetNodeInfo().GetHandle()); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "fail")}); + } + // const auto fh = (FHANDLE)FileHandle; + const auto fh = FileHandle.GetHandle(); + // DUMP(fh); + if (!fh) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "no filehandle")}); + } + + // if (fh) { + + /* + auto fh = ::open( + fname.c_str(), + r.GetNodeInfo().GetFlags() | O_CREAT, + r.GetNodeInfo().GetMode()); + DUMP(fh); + if (fh <= 0) { + DUMP("fail to create handle"); + return MakeFuture(TCompletedRequest{}); + } + */ + // OpenFiles[fname] = fh; + OpenHandles[fh] = std::move(FileHandle); + // TFileStat{}; + // const auto id = (FHANDLE)(fh); + // TFileStat stat{id}; + /* + TFileStat stat{fh}; + // GetStatByHandle(stat, id); + auto id = stat.INode; + + FileHandles[id] = std::move(fh); + */ + // DUMP(stat.INode, stat.IsDir(), stat.IsFile()); + + // DUMP(id, fh.GetLength(), fh.GetPosition()); + + // CreateIfMissing(PathByNode()) + HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; + // DUMP("savehandle", r.GetNodeInfo().GetHandle(), fh); + const auto inode = + TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; + if (r.GetNodeInfo().GetNodeId()) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; + + // NodePath[id] = PathByNode(parentNode) + // +r.GetNodeInfo().GetNewNodeName() +(isDir ? "/" : ""); + NodePath[inode] = relativePathName; + /* + DUMP( + "savepath2", + r.GetNodeInfo().GetNodeId(), + NodesLogToLocal[r.GetNodeInfo().GetNodeId()], + inode, + relativePathName); + */ + } + // NodePath[id] = PathByNode(parentNode) + // +r.GetNodeInfo().GetNewNodeName() +(isDir ? "/" : ""); + // DUMP(NodePath[id], parentNode); + STORAGE_DEBUG( + "open " << fh << "<-" << r.GetNodeInfo().GetHandle() + + << " known handles=" << HandlesLogToActual.size() + << " opened=" << OpenHandles.size() + << " inode=" << inode) + + ; + + // DUMP(NodePath); + // DUMP("open ok", HandlesLogToActual); + // return {}; + return MakeFuture( + TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); + } + + /* + static const int flags = + ProtoFlag(NProto::TCreateHandleRequest::E_CREATE) | + ProtoFlag(NProto::TCreateHandleRequest::E_EXCLUSIVE) | + ProtoFlag(NProto::TCreateHandleRequest::E_READ) | + ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); + */ + /* + auto name = GenerateNodeName(); + */ + auto request = CreateRequest(); + // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} + // nfs CreateHandle 0.004161s S_OK {parent_node_id=65, + // node_name=ini, flags=14, mode=436, node_id=66, + // handle=11024287581389312, size=0} + /* + request->SetNodeId(RootNodeId); + request->SetName(name); + request->SetFlags(flags); + */ + auto name = r.GetNodeInfo().GetNodeName(); + + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + // DUMP("nop", r.GetNodeInfo().GetParentNodeId()); + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + request->SetMode(r.GetNodeInfo().GetMode()); + + Cerr << "crrate=" << *request << "\n"; + + auto self = weak_from_this(); + return Session->CreateHandle(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr + ->HandleCreateHandle(future, name, started, r); + } + + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "cancelled")}); + }); + } + + TFuture HandleCreateHandle( + const TFuture& future, + const TString& name, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + auto handle = response.GetHandle(); + auto& infos = InitialFileSize ? IncompleteHandleInfos : HandleInfos; + with_lock (StateLock) { + TSegmentsPtr segments; + + /* + if (Spec.GetValidationEnabled()) { + segments = new TSegments(); + segments->resize(Spec.GetInitialFileSize() + / SEGMENT_SIZE); + } + */ + + infos.emplace_back(name, handle, 0, std::move(segments)); + HandleInfosh[handle] = {name, handle, 0, std::move(segments)}; + HandlesLogToActual[r.GetNodeInfo().GetHandle()] = handle; + } + + NThreading::TFuture setAttr; + if (InitialFileSize) { + static const int flags = + ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetNodeId(response.GetNodeAttr().GetId()); + request->SetFlags(flags); + request->MutableUpdate()->SetSize(InitialFileSize); + + setAttr = Session->SetNodeAttr( + CreateCallContext(), + std::move(request)); + } else { + setAttr = + NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); + } + + return setAttr.Apply( + [=, this](const TFuture& f) + { return HandleResizeAfterCreateHandle(f, name, started); }); + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "create handle for %s has failed: %s", + name.Quote().c_str(), + FormatError(error).c_str()); + + return NThreading::MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + error}); + } + } + + TCompletedRequest HandleResizeAfterCreateHandle( + const TFuture& future, + const TString& name, + TInstant started) + { + try { + const auto& response = future.GetValue(); + CheckResponse(response); + bool handleFound = false; + with_lock (StateLock) { + ui32 handleIdx = 0; + while (handleIdx < IncompleteHandleInfos.size()) { + auto& hinfo = IncompleteHandleInfos[handleIdx]; + if (hinfo.Name == name) { + handleFound = true; + + hinfo.Size = + Max(hinfo.Size, response.GetNode().GetSize()); + + STORAGE_INFO( + "updated file size for handle %lu and file %s" + ", size=%lu", + hinfo.Handle, + name.Quote().c_str(), + hinfo.Size); + + break; + } + + ++handleIdx; + } + + if (handleIdx < IncompleteHandleInfos.size()) { + DoSwap( + IncompleteHandleInfos[handleIdx], + IncompleteHandleInfos.back()); + + HandleInfos.push_back( + std::move(IncompleteHandleInfos.back())); + + IncompleteHandleInfos.pop_back(); + } + } + + if (!handleFound) { + STORAGE_WARN( + "handle for file %s not found", + name.Quote().c_str()); + } + + return {NProto::ACTION_CREATE_HANDLE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "set node attr handle for %s has failed: %s", + name.Quote().c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_HANDLE, started, error}; + } + } + + static constexpr ui32 BlockSize = 4_KB; + std::shared_ptr Acalloc(ui64 dataSize) + { + std::shared_ptr buffer = { + static_cast(aligned_alloc(BlockSize, dataSize)), + [](auto* p) + { + free(p); + }}; + memset(buffer.get(), 0, dataSize); + + return buffer; + } + TFuture DoReadData( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + if (Spec.GetNoRead()) { + return {}; + } + + TGuard guard(StateLock); + const auto started = TInstant::Now(); + if (UseFs()) { + const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); + // DUMP(handle, r.GetRanges(0).GetHandle()); + if (!handle) { + STORAGE_WARN( + "read: no handle " + << r.GetRanges(0).GetHandle() + << " ranges size=" << r.GetRanges().size() + << " map size=" << HandlesLogToActual.size()); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_FAIL, "cancelled")}); + } + // auto bytes = r.GetRanges(0).GetBytes(); + // auto buffer = NUnitTest::RandomString(bytes); + + // auto fh = TFileHandle{static_cast(handle)}; + auto& fh = OpenHandles[handle]; + STORAGE_DEBUG( + "Read from " << handle << " fh.len=" << fh.GetLength() + << " fh.pos=" << fh.GetPosition()); + // AsyncIO.Write( (FHANDLE)(handle), buffer.data(), bytes, + // r.GetRanges(0).GetOffset()); + auto buffer = Acalloc(r.GetRanges().cbegin()->GetBytes()); + + //[[maybe_unused]] const auto reserved = + fh.Reserve( + r.GetRanges().cbegin()->GetOffset() + + r.GetRanges().cbegin()->GetBytes()); + + // DUMP("gorrread",reserved,r.GetRanges().cbegin()->GetOffset(),r.GetRanges().cbegin()->GetBytes()); + TFileHandle FileHandle{fh.GetHandle()}; + + const auto future = AsyncIO.Read( + // fh, + FileHandle, + {}, + r.GetRanges().cbegin()->GetBytes(), + r.GetRanges().cbegin()->GetOffset()) + + //.Apply([](const auto& v){DUMP("rrrres", v.GetValue());}) + ; + FileHandle.Release(); + // fh.Release(); + + return future.Apply( + [started]([[maybe_unused]] const auto& future) mutable + { + // DUMP("writeresult", future.GetValue()); + return TCompletedRequest(NProto::ACTION_READ, started, {}); + }); + + // return MakeFuture(TCompletedRequest{NProto::ACTION_READ, started, + // {}}); + } + + // maybe before? + /* + if (HandleInfos.empty()) { + return DoCreateHandle(); + } + */ + // auto handleInfo = GetHandleInfo(); + /* + if (handleInfo.Size < ReadBytes) { + return DoWrite(handleInfo); + } + */ + + // return {}; + + /* + const ui64 slotOffset = PickSlot(handleInfo, ReadBytes); + const ui64 byteOffset = slotOffset * ReadBytes; + */ + auto request = CreateRequest(); + /* + request->SetHandle(handleInfo.Handle); + request->SetOffset(byteOffset); + request->SetLength(ReadBytes); + */ + // ensure exists + const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + request->SetHandle(handle); + request->SetOffset(r.GetRanges().cbegin()->GetOffset()); + request->SetLength(r.GetRanges().cbegin()->GetBytes()); + + auto self = weak_from_this(); + return Session->ReadData(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + return TCompletedRequest{ + NProto::ACTION_READ, + started, + future.GetValue().GetError()}; + /* + if (auto ptr = self.lock()) { + return ptr->HandleRead( + future, + handleInfo, + started, + byteOffset); + } + */ + return TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_FAIL, "cancelled")}; + }); + } + + TCompletedRequest HandleRead( + const TFuture& future, + THandleInfo handleInfo, + TInstant started + //,ui64 byteOffset + ) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + /* + const auto& buffer = response.GetBuffer(); + + // DUMP("read",buffer.Size(),TStringBuf(buffer, 0, + // std::min(10, buffer.size()))); + + ui64 segmentId = byteOffset / SEGMENT_SIZE; + for (ui64 offset = 0; offset < ReadBytes; offset += + SEGMENT_SIZE) { const TSegment* segment = reinterpret_cast(buffer.data() + offset); if + (Spec.GetValidationEnabled()) { + Y_ABORT_UNLESS(handleInfo.Segments); + + auto& segments = *handleInfo.Segments; + Y_ABORT_UNLESS(segmentId < segments.size()); + + TString message; + if (!Compare(segments[segmentId], *segment, + &message)) { throw TServiceError(E_FAIL) << Sprintf( "Validation + failed: %s", message.c_str()); + } + } + ++segmentId; + } + */ + + /* + with_lock (StateLock) { + HandleInfos.emplace_back(std::move(handleInfo)); + } + */ + + return {NProto::ACTION_READ, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "read for %s has failed: %s", + handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_READ, started, error}; + } + } + + TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) + { + TStringBuilder ret; + ret << start; + while (ret.size() < bytes) { + ret << " . " << offset + ret.size(); + } + return ret.substr(0, bytes); + } + + TFuture DoWrite( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r + // THandleInfo handleInfo = {} + ) + { + if (Spec.GetNoWrite()) { + return {}; + } + TGuard guard(StateLock); + const auto started = TInstant::Now(); + + if (UseFs()) { + const auto logHandle = r.GetRanges(0).GetHandle(); + const auto handle = HandleIdMapped(logHandle); + // DUMP(handle, r.GetRanges(0).GetHandle()); + if (!handle) { + return MakeFuture(TCompletedRequest( + NProto::ACTION_WRITE, + started, + MakeError( + E_CANCELLED, + TStringBuilder{} << "write cancelled: no handle =" + << logHandle))); // todo + } + const auto bytes = r.GetRanges(0).GetBytes(); + const auto offset = r.GetRanges(0).GetOffset(); + + TString buffer; + + if (Spec.GetWriteRandom()) { + buffer = NUnitTest::RandomString(bytes, logHandle); + } else if (Spec.GetWriteEmpty()) { + buffer = TString{bytes, ' '}; + } else { + buffer = MakeBuffer( + bytes, + offset, + TStringBuilder{} << "[ handle=" << logHandle + << " node=" << r.GetNodeInfo().GetNodeId() + << " bytes=" << bytes + << " offset=" << offset); + } + + // auto fh = TFileHandle{static_cast(handle)}; + + auto& fh = OpenHandles[handle]; + + STORAGE_DEBUG( + "Write to " << handle << " fh.length=" << fh.GetLength() + << " fh.pos=" << fh.GetPosition()); + // AsyncIO.Write( (FHANDLE)(handle), buffer.data(), bytes, + // r.GetRanges(0).GetOffset()); + // const auto fut = AsyncIO.Write(fh, buffer.data(), bytes, + // r.GetRanges(0).GetOffset()).Apply([](const auto& v) { + // DUMP("wwwwwwres", v.GetValue()); }); + // TODO TEST USE AFTER FREE on buffer + TFileHandle FileHandle{fh.GetHandle()}; + const auto writeFuture = AsyncIO.Write( + // fh, + FileHandle, + buffer.data(), + bytes, + offset); + FileHandle.Release(); + return writeFuture.Apply( + [started]([[maybe_unused]] const auto& future) mutable + { + // DUMP("writeresult", future.GetValue()); + return TCompletedRequest(NProto::ACTION_WRITE, started, {}); + }); + + // return MakeFuture(TCompletedRequest{}); + } + + /* + if (!handleInfo.Handle) { + if (HandleInfos.empty()) { + return DoCreateHandle(); + } + + handleInfo = GetHandleInfo(); + } + + const auto started = TInstant::Now(); + ui64 byteOffset = handleInfo.Size; + if (RandomNumber() < AppendProbability) { + handleInfo.Size += WriteBytes; + } else { + handleInfo.Size = Max(handleInfo.Size, WriteBytes); + const ui64 slotOffset = PickSlot(handleInfo, WriteBytes); + byteOffset = slotOffset * WriteBytes; + } + + auto request = CreateRequest(); + request->SetHandle(handleInfo.Handle); + request->SetOffset(byteOffset); + */ + // request->SetBuffer(); + auto request = CreateRequest(); + /*if (HandlesLogToActual.contains(r.GetRanges(0).GetHandle())) { + DUMP("unknown handle", r.GetRanges(0).GetHandle()); + } + request->SetHandle(HandlesLogToActual[r.GetRanges(0).GetHandle()]); +*/ + const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); + + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + request->SetHandle(handle); + + // auto byteOffset = r.GetRanges(0).GetOffset(); + request->SetOffset(r.GetRanges(0).GetOffset()); + auto bytes = r.GetRanges(0).GetBytes(); + //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} + // request.set + // TString buffer(bytes, '\0'); + // NUnitTest::RandomString() + // GenerateRandomString + // GenerateRandomData + auto buffer = NUnitTest::RandomString(bytes); + + *request->MutableBuffer() = std::move(buffer); + + ++LastWriteRequestId; + /* + TString buffer(WriteBytes, '\0'); + ui64 segmentId = byteOffset / SEGMENT_SIZE; + for (ui64 offset = 0; offset < WriteBytes; offset += + SEGMENT_SIZE) { TSegment* segment = + reinterpret_cast(buffer.begin() + offset); + segment->Handle = handleInfo.Handle; + segment->LastWriteRequestId = LastWriteRequestId; + + if (Spec.GetValidationEnabled()) { + Y_ABORT_UNLESS(handleInfo.Segments); + + auto& segments = *handleInfo.Segments; + if (segments.size() <= segmentId) { + segments.emplace_back(); + } + + segments[segmentId] = *segment; + } + + ++segmentId; + } + *request->MutableBuffer() = std::move(buffer); + */ + auto self = weak_from_this(); + return Session->WriteData(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleWrite( + future, + // handleInfo, + started); + } + + return TCompletedRequest{ + NProto::ACTION_WRITE, + started, + MakeError(E_FAIL, "cancelled")}; + }); + } + + TCompletedRequest HandleWrite( + const TFuture& future, + // THandleInfo handleInfo, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + with_lock (StateLock) { + // HandleInfos.emplace_back(std::move(handleInfo)); + } + + return {NProto::ACTION_WRITE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + /* + STORAGE_ERROR( + "write on %s has failed: %s", + handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + */ + return {NProto::ACTION_WRITE, started, error}; + } + } + + TString PathByNode(TNodeLocal nodeid) + { + if (const auto& it = NodePath.find(nodeid); it != NodePath.end()) { + return it->second; + } + return {}; + } + + /* + void CreateIfMissing(TString path) + { + TFsPath(path).MkDirs(); + } + */ + + static TNodeLocal Mkdir(const TString& name) + { + //[[maybe_unused]] const auto mkdires = + NFs::MakeDirectoryRecursive(name); + // mkdir(fname.c_str(), r.GetNodeInfo().GetMode()); + const auto inode = TFileStat{name}.INode; + // DUMP("mkdir", inode, name); + return inode; + } + + TFuture DoCreateNode( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + if (Spec.GetNoWrite()) { + return {}; + } + + TGuard guard(StateLock); + const auto started = TInstant::Now(); + + /* + auto name = GenerateNodeName(); + StagedNodes[name] = {}; + */ + if (UseFs()) { + // DUMP("========================================================"); + + auto parentNode = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + + parentNode = CreateDirIfMissingByNodeLog( + r.GetNodeInfo().GetNewParentNodeId()); + + // DUMP(parentNode, PathByNode(parentNode)); + + auto fname = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + + r.GetNodeInfo().GetNewNodeName(); + // DUMP(Spec.GetReplayRoot(), fname); + + ui64 nodeid = 0; + bool isDir = false; + switch (r.GetNodeInfo().GetType()) { + case NProto::E_REGULAR_NODE: { + // DUMP("file", fname); + + TFileHandle fh( + fname, + // static_cast(r.GetNodeInfo().GetMode()) + OpenAlways | RdWr // RdOnly + ); + fh.Reserve(r.GetNodeInfo().GetSize()); + // TFileStat{}; + // const auto id = (FHANDLE)(fh); + // TFileStat stat{id}; + if (fh) { + nodeid = TFileStat{fh}.INode; + } else { + nodeid = TFileStat{fname}.INode; + } + // GetStatByHandle(stat, id); + } break; + case NProto::E_DIRECTORY_NODE: { + isDir = true; + nodeid = Mkdir(fname); + // id = stat; + // DUMP("mkdir=", mkdires, id); + + // DUMP("dir"); + } break; + case NProto::E_LINK_NODE: + // NFs::HardLink(const TString &existingPath, const TString + // &newPath) NFs::SymLink(const TString &targetPath, const + // TString &linkPath) + // DUMP("link"); + break; + case NProto::E_SOCK_NODE: + // DUMP("sock"); + break; + case NProto::E_INVALID_NODE: + break; + } + + if (!nodeid) { + nodeid = TFileStat{fname}.INode; + } + + // DUMP(stat.INode, stat.IsDir(), stat.IsFile()); + + // DUMP(id, fh.GetLength(), fh.GetPosition()); + + // CreateIfMissing(PathByNode()) + if (nodeid) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = nodeid; + NodePath[nodeid] = PathByNode(parentNode) + + r.GetNodeInfo().GetNewNodeName() + + (isDir ? "/" : ""); + // DUMP("savepath", nodeid, NodePath[nodeid]); + + // DUMP(NodePath[id], parentNode); + } + + // DUMP(NodePath); + + return MakeFuture( + TCompletedRequest(NProto::ACTION_CREATE_HANDLE, started, {})); + } + /* + + + / 1 + /dir 2 p=1 + /dir/file 3 p=2 + + + + + + + + */ + auto request = CreateRequest(); + + // request->SetNodeId(r.GetNodeInfo().GetNodeId()); + // request->SetName(r.GetNodeInfo().GetNodeName()); + // if (r.GetNodeInfo().GetNewParentNodeId() == RootNodeId) { + // request->SetNodeId(r.GetNodeInfo().GetNewParentNodeId()); + //} else + + const auto parentNode = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + if (!parentNode) { + // DUMP("nop", r.GetNodeInfo().GetNewParentNodeId()); + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(parentNode); + // DUMP("pppath", PathByNode(parentNode)); + /* + if + (NodesLogToActual.contains(r.GetNodeInfo().GetNewParentNodeId())) { + request->SetNodeId( + NodesLogToActual[r.GetNodeInfo().GetNewParentNodeId()]); + } else { + DUMP( + "unknown parent node", + r.GetNodeInfo().GetNewParentNodeId(), + NodesLogToActual); + return MakeFuture(TCompletedRequest{}); + } + */ + auto name = r.GetNodeInfo().GetNewNodeName(); + request->SetName(r.GetNodeInfo().GetNewNodeName()); + + // request->SetGid(); + // request->SetUid(); + + // DUMP(r.GetNodeInfo().GetType()); + switch (r.GetNodeInfo().GetType()) { + case NProto::E_REGULAR_NODE: + // DUMP("file"); + request->MutableFile()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_DIRECTORY_NODE: + // DUMP("dir"); + request->MutableDirectory()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_LINK_NODE: + // DUMP("link"); + // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); + // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); + return MakeFuture(TCompletedRequest{}); // TODO + break; + + // case NProto::E_ symlink?: + // request->MutableSymlink()->SetTargetPath(); + + case NProto::E_SOCK_NODE: + // DUMP("sock"); + request->MutableSocket()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_INVALID_NODE: + // DUMP("invv?????"); + return MakeFuture( + TCompletedRequest{}); // Do not create files with invalid + // type - too hard to delete them + break; + } + + // request->Se + + // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, + // new_node_name=home, mode=509, node_id=12526, size=0} + + // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} + + // DUMP(request); + Cerr << "crreq=" << *request.get() << "\n"; + // auto started = TInstant::Now(); + auto self = weak_from_this(); + return Session->CreateNode(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleCreateNode(future, name, started, r); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleCreateNode( + const TFuture& future, + const TString& name, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + TGuard guard(StateLock); + // Y_ABORT_UNLESS(StagedNodes.erase(name)); + + try { + auto response = future.GetValue(); + + // DUMP("crd",name,r.GetNodeInfo().GetNodeId(),response.GetNode().GetId()); + + CheckResponse(response); + + Nodes[name] = TNode{name, response.GetNode()}; + // Nodes[name].Attrs.get + // NodesLogToActual[r.GetNodeInfo().GetNodeId()] = Nodes[name]; + if (response.GetNode().GetId()) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = + response.GetNode().GetId(); + + /* + DUMP( + "savepath4", + r.GetNodeInfo().GetNodeId(), + NodesLogToLocal[r.GetNodeInfo().GetNodeId()]); + */ + /* + const auto parentNode = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + NodePath[response.GetNode().GetId()] = + NodePath[parentNode] + "/" + + r.GetNodeInfo().GetNewNodeName(); + DUMP(NodePath[response.GetNode().GetId()], + parentNode); + */ + } + // DUMP(r.GetNodeInfo().GetNodeId(),response.GetNode().GetId()//,NodesLogToActual); + + return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "create node %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_NODE, started, error}; + } + } + + TFuture DoRenameNode( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + + { + if (Spec.GetNoRead()) { + return {}; + } + + TGuard guard(StateLock); + + if (UseFs()) { + const auto parentnodeid = + NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + + auto fname = Spec.GetReplayRoot() + + //"/" + + PathByNode(parentnodeid) + + r.GetNodeInfo().GetNodeName(); + + const auto newparentnodeid = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + + auto newfname = Spec.GetReplayRoot() + //"/" + + PathByNode(newparentnodeid) + + r.GetNodeInfo().GetNewNodeName(); + // NodePath[0]; + // DUMP("brename", TFileStat{fname}.Size, TFileStat{newfname}.Size); + const auto renameres = NFs::Rename(fname, newfname); + // DUMP("arename", TFileStat{fname}.Size, TFileStat{newfname}.Size); + // DUMP(renameres, fname, newfname); + STORAGE_DEBUG( + "rename " << fname << " => " << newfname << " : " << renameres); + return MakeFuture(TCompletedRequest{}); + } + + /* + if (Nodes.empty()) { + // return DoCreateNode({}); + } + + auto started = TInstant::Now(); + + auto it = Nodes.begin(); + auto old = it->first; + auto newbie = GenerateNodeName(); + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(old); + request->SetNewParentId(RootNodeId); + request->SetNewName(newbie); + + StagedNodes[newbie] = {}; + StagedNodes[old] = std::move(it->second); + Nodes.erase(it); + */ + + auto started = TInstant::Now(); + auto request = CreateRequest(); + // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} + // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, + // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} + // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetNewParentId( + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId())); + request->SetNewName(r.GetNodeInfo().GetNewNodeName()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + + // Cerr << "rename=" << *request << "\n"; + auto self = weak_from_this(); + return Session->RenameNode(CreateCallContext(), std::move(request)) + .Apply( + [= // , + + // old = std::move(old), newbie = std::move(newbie) + + ](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleRenameNode( + future, + // old, newbie, + + started, + r); + } + + return TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleRenameNode( + const TFuture& future, + // const TString& old, + // const TString& newbie, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + TGuard guard(StateLock); + /* + auto node = std::move(StagedNodes[old]); + StagedNodes.erase(old); + StagedNodes.erase(newbie); + */ + // Nodes.erase(r.GetNodeInfo().GetNodeName()); + try { + auto response = future.GetValue(); + CheckResponse(response); + Nodes[r.GetNodeInfo().GetNewNodeName()] = + std::move(Nodes[r.GetNodeInfo().GetNodeName()]); + Nodes.erase(r.GetNodeInfo().GetNodeName()); + // Nodes[newbie] = std::move(node); + return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "rename node %s has failed: %s", + r.GetNodeInfo().GetNodeName().c_str(), // old.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_RENAME_NODE, started, error}; + } + } + + TFuture DoUnlinkNode( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + if (Spec.GetNoWrite()) { + return {}; + } + + // UnlinkNode 0.002605s S_OK {parent_node_id=3, + // node_name=tfrgYZ1} + + TGuard guard(StateLock); + auto started = TInstant::Now(); + + if (UseFs()) { + // DUMP(NodePath); + const auto parentNodeId = + NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!parentNodeId) { + STORAGE_WARN( + "unlink : no parent orig=" + << r.GetNodeInfo().GetParentNodeId()); + return MakeFuture(TCompletedRequest( + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_CANCELLED, "cancelled"))); + } + const auto fname = Spec.GetReplayRoot() + "/" + + PathByNode(parentNodeId) + + r.GetNodeInfo().GetNodeName(); + // DUMP(r.GetNodeInfo().GetParentNodeId(),PathByNode(NodeIdMapped(r.GetNodeInfo().GetParentNodeId()))); + + // {parent_node_id=3, node_name=tfrgYZ1} + // const auto unlinkres = unlink(fname.c_str()); + const auto unlinkres = NFs::Remove(fname); + STORAGE_DEBUG("unlink " << fname << " = " << unlinkres); + // DUMP(unlinkres, fname); + // TODO : + // NodesLogToActual.erase(...) + // NodePath.erase(...) + return MakeFuture( + TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); + } + + /* + if (Nodes.empty()) { + // return DoCreateNode(); + } + */ + + /* + auto it = Nodes.begin(); + auto name = it->first; + */ + + auto name = r.GetNodeInfo().GetNodeName(); + auto request = CreateRequest(); + // request->SetNodeId(RootNodeId); + request->SetName(name); + // request.set + // request->SetNodeId(r.GetNodeInfo().GetNodeId()); + // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + /* + StagedNodes[name] = std::move(it->second); + Nodes.erase(it); + */ + auto self = weak_from_this(); + return Session->UnlinkNode(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleUnlinkNode(future, name, started); + } + + return TCompletedRequest{ + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleUnlinkNode( + const TFuture& future, + const TString& name, + TInstant started) + { + with_lock (StateLock) { + StagedNodes.erase(name); + } + + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "unlink for %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_REMOVE_NODE, started, error}; + } + } + /* + TFuture DoCreateHandle2() + { + static const int flags = + ProtoFlag(NProto::TCreateHandleRequest::E_READ) | + ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); + + TGuard guard(StateLock); + if (Nodes.empty()) { + // return DoCreateNode(); + } + + auto started = TInstant::Now(); + + auto it = Nodes.begin(); + if (Nodes.size() > 1) { + std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); + } + + auto name = it->first; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + request->SetFlags(flags); + + auto self = weak_from_this(); + return Session->CreateHandle(CreateCallContext(), + std::move(request)) .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleCreateHandle2(future, name, + started); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleCreateHandle2( + const TFuture& future, + const TString name, + TInstant started) + { + TGuard guard(StateLock); + + try { + auto response = future.GetValue(); + CheckResponse(response); + + auto handle = response.GetHandle(); + Handles[handle] = THandle{name, handle}; + + return {NProto::ACTION_CREATE_HANDLE, started, + response.GetError()}; } catch (const TServiceError& e) { auto error = + MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "create + handle for %s has failed: %s", name.c_str(), FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_HANDLE, started, error}; + } + } + */ + TFuture DoDestroyHandle( + NCloud::NFileStore::NProto::TProfileLogRequestInfo r) + { + // DestroyHandle 0.002475s S_OK {node_id=10, + // handle=61465562388172112} + TGuard guard(StateLock); + auto started = TInstant::Now(); + + if (UseFs()) { + const auto handleid = HandleIdMapped(r.GetNodeInfo().GetHandle()); + + const auto& it = OpenHandles.find(handleid); + if (it == OpenHandles.end()) { + return MakeFuture(TCompletedRequest( + NProto::ACTION_DESTROY_HANDLE, + started, + MakeError( + E_CANCELLED, + TStringBuilder{} << "close " << handleid << " <- " + << r.GetNodeInfo().GetHandle() + << " fail: not found in " + << OpenHandles.size()))); + } + + auto& fhandle = it->second; + const auto len = fhandle.GetLength(); + const auto pos = fhandle.GetPosition(); + // auto closed = + fhandle.Close(); + OpenHandles.erase(handleid); + HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); + STORAGE_DEBUG( + "Close " << handleid << " orig=" + << r.GetNodeInfo().GetHandle() + //<< " closed=" << closed + << " pos=" << pos << " len=" << len + << " open map size=" << OpenHandles.size() + << " map size=" << HandlesLogToActual.size()); + return MakeFuture( + TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, started, {})); + } + + /* + if (Handles.empty()) { + return DoCreateHandle(); + } + + auto started = TInstant::Now(); + auto it = Handles.begin(); + + ui64 handle = it->first; + auto name = it->second.Path; + Handles.erase(it); + if (auto it = Locks.find(handle); it != Locks.end()) { + Locks.erase(it); + } + if (auto it = StagedLocks.find(handle); it != StagedLocks.end()) + { StagedLocks.erase(it); + } + + auto request = CreateRequest(); + request->SetHandle(handle); + */ + auto name = r.GetNodeInfo().GetNodeName(); + + auto request = CreateRequest(); + + const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + + HandlesLogToActual.erase(handle); + + request->SetHandle(handle); + + auto self = weak_from_this(); + return Session->DestroyHandle(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleDestroyHandle(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_DESTROY_HANDLE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TFuture DoGetNodeAttr( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + if (Spec.GetNoRead()) { + return {}; + } + + TGuard guard(StateLock); + const auto started = TInstant::Now(); + + // TODO: by parent + name // + // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} + + // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, + // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} + + if (UseFs()) { + if (r.GetNodeInfo().GetNodeName()) { + KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name = + r.GetNodeInfo().GetNodeName(); + /* + DUMP( + "saveknown1", + r.GetNodeInfo().GetNodeId(), + r.GetNodeInfo().GetNodeName()); + */ + } + if (r.GetNodeInfo().GetParentNodeId() && + r.GetNodeInfo().GetParentNodeId() != + r.GetNodeInfo().GetNodeId()) + { + KnownLogNodes[r.GetNodeInfo().GetNodeId()].ParentLog = + r.GetNodeInfo().GetParentNodeId(); + /* + DUMP( + "saveknown2", + r.GetNodeInfo().GetNodeId(), + r.GetNodeInfo().GetParentNodeId()); + */ + } + + // TODO: can create and truncate to size here missing files + + // DUMP(KnownNodes); + + const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + + if (!nodeid) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_CANCELLED, "cancelled")}); + } + + auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); + [[maybe_unused]] const auto stat = TFileStat{fname}; + // DUMP("gna", fname, stat.INode); + + return MakeFuture( + TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); + } + /* + if (Nodes.empty()) { + //return DoCreateNode(); + } + */ + /* + auto it = Nodes.begin(); + if (Nodes.size() > 1) { + std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); + } + + auto name = it->first; + + auto request = CreateRequest(); + request->SetNodeId(RootNodeId); + request->SetName(name); + */ + + /* + if + (!NodesLogToActual.contains(r.GetNodeInfo().GetParentNodeId())) { + DUMP("skip nodeattr"); + return MakeFuture(TCompletedRequest{}); + } + */ + auto request = CreateRequest(); + // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + auto name = r.GetNodeInfo().GetNodeName(); + request->SetName(r.GetNodeInfo().GetNodeName()); + // request->SetFlags(r.GetNodeInfo().GetFlags()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + /* + DUMP( + "ids?", + r.GetNodeInfo().GetParentNodeId(), + NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); + if (!NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]) { + DUMP(NodesLogToActual); + } + */ + Cerr << "attr=" << *request << "\n"; + auto self = weak_from_this(); + STORAGE_DEBUG("GetNodeAttr client started"); + return Session->GetNodeAttr(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleGetNodeAttr(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleGetNodeAttr( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + STORAGE_DEBUG("GetNodeAttr client completed"); + CheckResponse(response); + TGuard guard(StateLock); + /* + if (response.GetNode().SerializeAsString() != + Nodes[name].Attrs.SerializeAsString()) + { + auto error = MakeError( + E_FAIL, + TStringBuilder() + << "attributes are not equal for node " << + name << ": " + << response.GetNode().DebugString().Quote() + << " != " << + Nodes[name].Attrs.DebugString().Quote()); + STORAGE_ERROR(error.GetMessage().c_str()); + } + */ + return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "get node attr %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + // return {NProto::ACTION_GET_NODE_ATTR, started, error}; + } + } + + TCompletedRequest HandleDestroyHandle( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_DESTROY_HANDLE, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "destroy handle %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_DESTROY_HANDLE, started, error}; + } + } + + TFuture DoAcquireLock() + { + TGuard guard(StateLock); + if (Handles.empty()) { + // return DoCreateHandle(); + } + + auto started = TInstant::Now(); + auto it = Handles.begin(); + while (it != Handles.end() && + (Locks.contains(it->first) || StagedLocks.contains(it->first))) + { + ++it; + } + + if (it == Handles.end()) { + // return DoCreateHandle(); + } + + auto handle = it->first; + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto self = weak_from_this(); + return Session->AcquireLock(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleAcquireLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleAcquireLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it == StagedLocks.end()) { + // nothing todo, file was removed + Y_ABORT_UNLESS(!Locks.contains(handle)); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } + + StagedLocks.erase(it); + + try { + const auto& response = future.GetValue(); + CheckResponse(response); + + Locks.insert(handle); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "acquire lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_ACQUIRE_LOCK, started, error}; + } + } + + TFuture DoReleaseLock() + { + TGuard guard(StateLock); + if (Locks.empty()) { + return DoAcquireLock(); + } + + auto it = Locks.begin(); + auto handle = *it; + + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + Locks.erase(it); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto started = TInstant::Now(); + auto self = weak_from_this(); + return Session->ReleaseLock(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleReleaseLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleReleaseLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it != StagedLocks.end()) { + StagedLocks.erase(it); + } + + try { + CheckResponse(future.GetValue()); + return {NProto::ACTION_RELEASE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "release lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_RELEASE_LOCK, started, error}; + } + } + + template + std::shared_ptr CreateRequest() + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->CopyFrom(Headers); + + return request; + } + + TString GenerateNodeName() + { + return TStringBuilder() + << Headers.GetClientId() << ":" << CreateGuidAsString(); + } + + THandleInfo GetHandleInfo() + { + Y_ABORT_UNLESS(!HandleInfos.empty()); + ui64 index = RandomNumber(HandleInfos.size()); + std::swap(HandleInfos[index], HandleInfos.back()); + + auto handle = HandleInfos.back(); + HandleInfos.pop_back(); + + return handle; + } + + THandleInfo GetHandleInfo(ui64 id) + { + return HandleInfos[id]; + /* + Y_ABORT_UNLESS(!HandleInfos.empty()); + ui64 index = RandomNumber(HandleInfos.size()); + std::swap(HandleInfos[index], HandleInfos.back()); + + auto handle = HandleInfos.back(); + HandleInfos.pop_back(); + + return handle; + */ + } + + template + void CheckResponse(const T& response) + { + if (HasError(response)) { + throw TServiceError(response.GetError()); + } + } + + TIntrusivePtr CreateCallContext() + { + return MakeIntrusive( + LastRequestId.fetch_add(1, std::memory_order_relaxed)); + } + /* + ui64 PickSlot(THandleInfo& handleInfo, ui64 reqBytes) + { + const ui64 slotCount = handleInfo.Size / reqBytes; + Y_ABORT_UNLESS(slotCount); + if (Spec.GetSequential()) { + return handleInfo.LastSlot = (handleInfo.LastSlot + 1) % + slotCount; + } + + return RandomNumber(slotCount); + } + */ +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IRequestGeneratorPtr CreateReplayRequestGenerator( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) +{ + return std::make_shared( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)); +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp new file mode 100644 index 0000000000..09fd751480 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -0,0 +1,1156 @@ +#include "test.h" + +#include "client.h" +#include "context.h" +#include "dump.h" +#include "request.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NReplay { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +struct TTestStats +{ + struct TStats + { + ui64 Requests = 0; + TLatencyHistogram Hist; + }; + + TString Name; + bool Success = true; + TMap ActionStats; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template +T WaitForCompletion(const TString& request, const TFuture& future) +{ + const auto& response = future.GetValue(TDuration::Max()); + if (HasError(response)) { + const auto& error = response.GetError(); + throw yexception() << "Failed to execute " << request + << " request: " << FormatError(error); + } + + return response; +} + +//////////////////////////////////////////////////////////////////////////////// + +class TRequestsCompletionQueue +{ +private: + TAdaptiveLock Lock; + TDeque> Items; + +public: + void Enqueue(std::unique_ptr request) + { + with_lock (Lock) { + Items.emplace_back(std::move(request)); + } + } + + std::unique_ptr Dequeue() + { + with_lock (Lock) { + std::unique_ptr ptr; + if (Items) { + ptr = std::move(Items.front()); + Items.pop_front(); + } + + return ptr; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TSetupSession +{ + bool ReadOnly = false; + ui64 SessionSeqNo = 0; + TPromise Complete; +}; + +struct TTeardown +{ + TPromise Complete; +}; +struct TSuspendLoad +{ +}; +struct TResumeLoad +{ +}; +struct TProcessCompletedRequests +{ +}; + +using TLoadTestCommand = std::variant< + TTeardown, + TSuspendLoad, + TResumeLoad, + TProcessCompletedRequests, + TSetupSession>; + +//////////////////////////////////////////////////////////////////////////////// + +class TLoadTestCommandChannel +{ +private: + TAdaptiveLock Lock; + TDeque Items; + bool HasProcessCompletedRequests = false; + +public: + void Enqueue(TLoadTestCommand command) + { + with_lock (Lock) { + if (holds_alternative(command)) { + if (HasProcessCompletedRequests) { + return; + } + HasProcessCompletedRequests = true; + } + Items.emplace_back(std::move(command)); + } + } + + TMaybe Dequeue() + { + with_lock (Lock) { + if (Items) { + auto value = std::move(Items.front()); + Items.pop_front(); + if (holds_alternative(value)) { + HasProcessCompletedRequests = false; + } + + return value; + } + + return {}; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TStartSource +{ +}; + +struct TStartTarget +{ +}; + +struct TFinishMigration +{ +}; + +struct TTestFinished +{ + TString Id; +}; + +using TLoadTestControllerCommand = + std::variant; + +//////////////////////////////////////////////////////////////////////////////// + +class TLoadTestControllerCommandChannel +{ +private: + TAdaptiveLock Lock; + TDeque Items; + TAutoEvent Event; + +public: + void Enqueue(TLoadTestControllerCommand command) + { + with_lock (Lock) { + Items.emplace_back(std::move(command)); + } + Event.Signal(); + } + + TMaybe Dequeue() + { + with_lock (Lock) { + if (Items) { + auto value = std::move(Items.front()); + Items.pop_front(); + return value; + } + + return {}; + } + } + + void Wait() + { + Event.WaitI(); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +class TLoadTest final + : public ITest + , public ISimpleThread + , public std::enable_shared_from_this +{ +private: + static constexpr TDuration ReportInterval = TDuration::Seconds(5); + + const NProto::TLoadTest Config; + const ITimerPtr Timer; + const ISchedulerPtr Scheduler; + const ILoggingServicePtr Logging; + const TString TestInstanceId; + const TString TestTag; + + TLoadTestControllerCommandChannel& Controller; + + TLog Log; + IFileStoreServicePtr Client; + ISessionPtr Session; + + TString FileSystemId; + TString ClientId; + TString SessionId; + + ui32 MaxIoDepth = 64; + i32 CurrentIoDepth = 0; + + TDuration MaxDuration; + TInstant StartTs; + + ui64 MaxRequests = 0; + ui64 RequestsSent = 0; + ui64 RequestsCompleted = 0; + + TInstant LastReportTs; + ui64 LastRequestsCompleted = 0; + + IRequestGeneratorPtr RequestGenerator; + TRequestsCompletionQueue CompletionQueue; + TLoadTestCommandChannel Commands; + TAutoEvent Event; + + bool LoadEnabled = true; + + ui64 SeqNo = 0; + + TTestStats TestStats; + TPromise Result = NewPromise(); + + bool ShutdownFlag = false; + +public: + TLoadTest( + const NProto::TLoadTest& config, + ITimerPtr timer, + ISchedulerPtr scheduler, + ILoggingServicePtr logging, + IClientFactoryPtr clientFactory, + TString testInstanceId, + TLoadTestControllerCommandChannel& controller, + TString clientId = {}) + : Config(config) + , Timer(std::move(timer)) + , Scheduler(std::move(scheduler)) + , Logging(std::move(logging)) + , TestInstanceId(std::move(testInstanceId)) + , TestTag(NReplay::MakeTestTag(Config.GetName())) + , Controller(controller) + , Client(clientFactory->CreateClient()) + , ClientId(std::move(clientId)) + { + Log = Logging->CreateLog("RUNNER"); + } + + TFuture Run() override + { + ISimpleThread::Start(); + return Result; + } + + void GenerateRequests() + { + if (LoadEnabled && !ShouldStop()) { + while (SendNextRequest()) { + //DUMP("rb", RequestsSent); + ++RequestsSent; + STORAGE_DEBUG( + "%s request sent: %s %lu %lu", + MakeTestTag().c_str(), + SessionId.c_str(), + SeqNo, RequestsSent); + //DUMP("re"); + } + DUMP("fin"); +//PrintBackTrace(); + } + } + + void EnqueueCommand(TLoadTestCommand command) + { + Commands.Enqueue(std::move(command)); + Event.Signal(); + } + + void SetupSession(TSetupSession& cmd) + { + if (!ShouldStop()) { + STORAGE_INFO( + "%s establishing session %lu", + MakeTestTag().c_str(), + cmd.SessionSeqNo); + + TFuture result; + if (!SessionId) { + result = Session->CreateSession(cmd.ReadOnly, cmd.SessionSeqNo); + } else { + result = Session->AlterSession(cmd.ReadOnly, cmd.SessionSeqNo); + } + + WaitForCompletion("create-session", result); + auto response = result.GetValue(); + if (FAILED(response.GetError().GetCode())) { + STORAGE_INFO( + "%s failed establish session: %s %lu %s", + MakeTestTag().c_str(), + SessionId.c_str(), + cmd.SessionSeqNo, + response.GetError().GetMessage().c_str()); + TestStats.Success = false; + cmd.Complete.SetValue(false); + return; + } + auto sessionId = response.GetSession().GetSessionId(); + if (SessionId && SessionId != sessionId) { + STORAGE_INFO( + "%s session established but id has changed: %s %lu %s", + MakeTestTag().c_str(), + SessionId.c_str(), + cmd.SessionSeqNo, + sessionId.c_str()); + TestStats.Success = false; + cmd.Complete.SetValue(false); + return; + } + SessionId = sessionId; + STORAGE_INFO( + "%s session established: %s %lu", + MakeTestTag().c_str(), + SessionId.c_str(), + cmd.SessionSeqNo); + } + + cmd.Complete.SetValue(true); + } + + bool HandleTeardown(TTeardown& cmd) + { + STORAGE_INFO("%s got teardown", MakeTestTag().c_str()) + TeardownTest(cmd); + return false; + } + + bool HandleSuspendLoad(const TSuspendLoad&) + { + LoadEnabled = false; + return true; + } + + bool HandleResumeLoad(const TResumeLoad&) + { + LoadEnabled = true; + GenerateRequests(); + return true; + } + + bool HandleProcessCompletedRequests(const TProcessCompletedRequests&) + { + ProcessCompletedRequests(); + ReportProgress(); + GenerateRequests(); + return true; + } + + bool HandleSetupSession(TSetupSession& cmd) + { + SeqNo = cmd.SessionSeqNo; + SetupSession(cmd); + GenerateRequests(); + return true; + } + + bool HandleCommand(TLoadTestCommand cmd) + { + return std::visit( + TOverloaded{ + [this](TTeardown& cmd) { return HandleTeardown(cmd); }, + [this](const TSuspendLoad& cmd) + { return HandleSuspendLoad(cmd); }, + [this](const TResumeLoad& cmd) + { return HandleResumeLoad(cmd); }, + [this](const TProcessCompletedRequests& cmd) + { return HandleProcessCompletedRequests(cmd); }, + [this](TSetupSession& cmd) + { + return HandleSetupSession(cmd); + }}, + cmd); + } + + void* ThreadProc() override + { + StartTs = TInstant::Now(); + NCloud::SetCurrentThreadName(MakeTestTag()); + + try { + SetupTest(); + LastReportTs = TInstant::Now(); + + for (;;) { + Event.WaitI(); + bool cont = true; + while (auto maybeCmd = Commands.Dequeue()) { + auto& cmd = *maybeCmd; + if (cont = HandleCommand(cmd); !cont) { + break; + } + } + if (ShouldStop() && !ShutdownFlag) { + TTeardown dummy; + TeardownTest(dummy); + + TTestFinished cmd; + cmd.Id = TestInstanceId; + Controller.Enqueue(cmd); + ShutdownFlag = true; + } + if (!cont) { + break; + } + } + + // prevent race between this thread and main thread + // destroying test instance right after setvalue() + auto result = Result; + result.SetValue(GetStats()); + } catch (...) { + STORAGE_ERROR( + "%s test has failed: %s", + MakeTestTag().c_str(), + CurrentExceptionMessage().c_str()); + + Result.SetException(std::current_exception()); + } + + return nullptr; + } + + TString GetTestId() const + { + return TestInstanceId; + } + +private: + void SetupTest() + { + Client->Start(); + + // setup test limits + if (auto depth = Config.GetIODepth()) { + MaxIoDepth = depth; + } + + MaxRequests = Config.GetRequestsCount(); + MaxDuration = TDuration::Seconds(Config.GetTestDuration()); + + STORAGE_INFO("%s setting up test", MakeTestTag().c_str()); + + // setup test filestore + if (Config.HasFileSystemId()) { + FileSystemId = Config.GetFileSystemId(); + } else if (Config.HasCreateFileStoreRequest()) { + FileSystemId = Config.GetCreateFileStoreRequest().GetFileSystemId(); + } + + CreateSession(); + + NProto::THeaders headers; + headers.SetClientId(Config.GetName()); + headers.SetSessionId(SessionId); + + DUMP(Config); + + switch (Config.GetSpecsCase()) { + case NProto::TLoadTest::kReplaySpec: + // DUMP("repl", Config.GetReplaySpec().GetFileName()); + RequestGenerator = CreateReplayRequestGenerator( + Config.GetReplaySpec(), + Logging, + Session, + FileSystemId, + headers); + break; + + default: + ythrow yexception() + << MakeTestTag() << " config should have test spec"; + } + } + + void CreateSession() + { + NProto::TSessionConfig proto; + proto.SetFileSystemId(FileSystemId); + proto.SetClientId(ClientId); + proto.SetSessionPingTimeout(Config.GetSessionPingTimeout()); + proto.SetSessionRetryTimeout(Config.GetSessionRetryTimeout()); + + Session = NClient::CreateSession( + Logging, + Timer, + Scheduler, + Client, + std::make_shared(proto)); + } + + bool ShouldStop() const + { + return !TestStats.Success || LimitsReached(); + } + + bool LimitsReached() const + { + return (MaxDuration && TInstant::Now() - StartTs >= MaxDuration) || + (MaxRequests && RequestsSent >= MaxRequests); + } + + bool SendNextRequest() + { + if (LimitsReached() || + //(MaxIoDepth && CurrentIoDepth >= MaxIoDepth) || + !RequestGenerator->HasNextRequest() || ShouldStop()) + { + DUMP("nop"); + return false; + } + + ++CurrentIoDepth; + auto self = weak_from_this(); + const auto future = RequestGenerator->ExecuteNextRequest(); + // DUMP(future.Initialized(), CurrentIoDepth); + if (!future.Initialized()) { + return true; + } + future.Apply( + [=](const TFuture& future) + { + if (!future.Initialized()) { + return; + } + if (auto ptr = self.lock()) { + ptr->SignalCompletion(future.GetValue()); + } + }); + return true; + } + + void SignalCompletion(TCompletedRequest request) + { + CompletionQueue.Enqueue( + std::make_unique(std::move(request))); + + Commands.Enqueue(TProcessCompletedRequests{}); + Event.Signal(); + } + + void ProcessCompletedRequests() + { + while (auto request = CompletionQueue.Dequeue()) { + Y_ABORT_UNLESS(CurrentIoDepth > 0); + --CurrentIoDepth; + ++RequestsCompleted; + + auto code = request->Error.GetCode(); + if (FAILED(code)) { + STORAGE_WARN( + "%s failed request: %s", + MakeTestTag().c_str(), + FormatError(request->Error).c_str()); + + // TestStats.Success = false; + } + + if (request->Stop) { + TestStats.Success = false; + } + auto& stats = TestStats.ActionStats[request->Action]; + ++stats.Requests; + stats.Hist.RecordValue(request->Elapsed); + } + // DUMP("compl"); + } + + void ReportProgress() + { + auto now = TInstant::Now(); + auto elapsed = now - LastReportTs; + + if (elapsed > ReportInterval) { + const auto requestsCompleted = + RequestsCompleted - LastRequestsCompleted; + + auto stats = GetStats(); + STORAGE_INFO( + "%s current rate: %ld r/s; stats:\n%s", + MakeTestTag().c_str(), + (ui64)(requestsCompleted / elapsed.Seconds()), + NProtobufJson::Proto2Json(stats, {.FormatOutput = true}) + .c_str()); + + LastReportTs = now; + LastRequestsCompleted = RequestsCompleted; + } + } + + NProto::TTestStats GetStats() + { + NProto::TTestStats results; + results.SetName(Config.GetName()); + results.SetSuccess(TestStats.Success); + + auto* stats = results.MutableStats(); + for (const auto& pair: TestStats.ActionStats) { + auto* action = stats->Add(); + action->SetAction(NProto::EAction_Name(pair.first)); + action->SetCount(pair.second.Requests); + FillLatency(pair.second.Hist, *action->MutableLatency()); + } + + // TODO report some stats for the files generated during the shooting + + return results; + } + + void TeardownTest(TTeardown& cmd) + { + DUMP("teardown", ShutdownFlag, CurrentIoDepth); + if (ShutdownFlag) { + if (cmd.Complete.Initialized()) { + cmd.Complete.SetValue(true); + } + return; + } + + STORAGE_INFO("%s tear down", MakeTestTag().c_str()); + while (CurrentIoDepth > 0) { + DUMP("td io", CurrentIoDepth); + Sleep(TDuration::Seconds(1)); + ProcessCompletedRequests(); + } + DUMP("tdok"); + auto ctx = MakeIntrusive(); + + auto result = Session->DestroySession(); + WaitForCompletion("destroy session", result); + + if (Client) { + Client->Stop(); + } + + auto response = result.GetValue(); + if (FAILED(response.GetError().GetCode())) { + STORAGE_INFO( + "%s failed to destroy session: %s %lu %s", + MakeTestTag().c_str(), + SessionId.c_str(), + SeqNo, + response.GetError().GetMessage().c_str()); + TestStats.Success = false; + } + + if (cmd.Complete.Initialized()) { + cmd.Complete.SetValue(SUCCEEDED(response.GetError().GetCode())); + } + } + + void FillLatency(const TLatencyHistogram& hist, NProto::TLatency& latency) + { + latency.SetP50(hist.GetValueAtPercentile(50)); + latency.SetP90(hist.GetValueAtPercentile(90)); + latency.SetP95(hist.GetValueAtPercentile(95)); + latency.SetP99(hist.GetValueAtPercentile(99)); + latency.SetP999(hist.GetValueAtPercentile(99.9)); + latency.SetMin(hist.GetMin()); + latency.SetMax(hist.GetMax()); + latency.SetMean(hist.GetMean()); + latency.SetStdDeviation(hist.GetStdDeviation()); + } + + const TString& MakeTestTag() const + { + return TestTag; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +std::shared_ptr CreateSimpleLoadTest( + const NProto::TLoadTest& config, + ITimerPtr timer, + ISchedulerPtr scheduler, + ILoggingServicePtr logging, + IClientFactoryPtr clientFactory, + TString testInstanceId, + TLoadTestControllerCommandChannel& controller, + TString clientId) +{ + return std::make_shared( + config, + std::move(timer), + std::move(scheduler), + std::move(logging), + std::move(clientFactory), + std::move(testInstanceId), + controller, + std::move(clientId)); +} + +//////////////////////////////////////////////////////////////////////////////// + +class TLoadTestController final + : public ITest + , public ISimpleThread + , public std::enable_shared_from_this +{ +private: + const TAppContext& Ctx; + const NProto::TLoadTest Config; + const ITimerPtr Timer; + const ISchedulerPtr Scheduler; + const ILoggingServicePtr Logging; + const IClientFactoryPtr ClientFactory; + const TString TestTag; + + TLog Log; + std::shared_ptr SourceTest; + std::shared_ptr TargetTest; + TFuture SourceFuture; + TFuture TargetFuture; + + TDuration MaxDuration; + + TInstant StartTs; + + TLoadTestControllerCommandChannel Events; + + TTestStats TestStats; + TPromise Result = NewPromise(); + + ui64 SessionSeqNo = 0; + + IFileStoreServicePtr Client; + +public: + TLoadTestController( + const TAppContext& ctx, + const NProto::TLoadTest& config, + ITimerPtr timer, + ISchedulerPtr scheduler, + ILoggingServicePtr logging, + IClientFactoryPtr clientFactory) + : Ctx(ctx) + , Config(config) + , Timer(std::move(timer)) + , Scheduler(std::move(scheduler)) + , Logging(std::move(logging)) + , ClientFactory(std::move(clientFactory)) + , TestTag(NReplay::MakeTestTag("Controller_" + Config.GetName())) + , Client(ClientFactory->CreateClient()) + { + Log = Logging->CreateLog("TEST_CONTROLLER"); + } + + TFuture Run() override + { + Events.Enqueue(TStartSource{}); + + ISimpleThread::Start(); + + return Result; + } + + TFuture SendSetupTestSession( + std::shared_ptr& test, + bool readOnly, + ui64 sessionSeqNo) + { + TPromise complete = NewPromise(); + + TSetupSession cmd; + cmd.ReadOnly = readOnly; + cmd.SessionSeqNo = sessionSeqNo; + cmd.Complete = complete; + test->EnqueueCommand(std::move(cmd)); + + return complete.GetFuture(); + } + + TFuture SendTeardownTest(std::shared_ptr& test) + { + TPromise complete = NewPromise(); + TTeardown teardown; + teardown.Complete = complete; + test->EnqueueCommand(teardown); + + return complete.GetFuture(); + } + + bool HandleStartSource(const TStartSource&) + { + auto testId = CreateGuidAsString(); + STORAGE_ERROR( + "%s start test %s", + MakeTestTag().c_str(), + testId.c_str()); + + SourceTest = CreateSimpleLoadTest( + Config, + Timer, + Scheduler, + Logging, + ClientFactory, + std::move(testId), + Events, + "migration-test"); + SourceFuture = SourceTest->Run(); + + auto result = SendSetupTestSession(SourceTest, false, SessionSeqNo); + if (!result.GetValueSync()) { + return false; + } + + auto period = Config.GetMigrationSpec().GetMigrationPeriod(); + if (period) { + Scheduler->Schedule( + Timer->Now() + TDuration::Seconds(period), + [weakPtr = weak_from_this()] + { + if (auto self = weakPtr.lock()) { + self->Events.Enqueue(TStartTarget{}); + } + }); + } + return true; + } + + bool HandleStartTarget(const TStartTarget&) + { + auto testId = CreateGuidAsString(); + STORAGE_INFO( + "%s start a new client %s", + MakeTestTag().c_str(), + testId.c_str()); + + TargetTest = CreateSimpleLoadTest( + Config, + Timer, + Scheduler, + Logging, + ClientFactory, + std::move(testId), + Events, + "migration-test"); + TargetFuture = TargetTest->Run(); + + auto result = SendSetupTestSession(TargetTest, true, SessionSeqNo + 1); + if (!result.GetValueSync()) { + return false; + } + + auto period = Config.GetMigrationSpec().GetStateTransferDelay(); + if (period) { + Scheduler->Schedule( + Timer->Now() + TDuration::Seconds(period), + [weakPtr = weak_from_this()] + { + if (auto self = weakPtr.lock()) { + self->Events.Enqueue(TFinishMigration{}); + } + }); + } + return true; + } + + bool HandleFinishMigration(const TFinishMigration&) + { + STORAGE_INFO( + "%s switch to new client %s", + MakeTestTag().c_str(), + TargetTest->GetTestId().c_str()); + + auto source = SendSetupTestSession(SourceTest, true, SessionSeqNo); + auto target = SendSetupTestSession(TargetTest, false, ++SessionSeqNo); + + TVector> futures{source, target}; + WaitAll(futures).Wait(); + if (!source.GetValue() || !target.GetValue()) { + STORAGE_ERROR( + "%s failed to switch to new client", + MakeTestTag().c_str()); + return false; + } + + TTeardown teardown; + SourceTest->EnqueueCommand(teardown); + SourceFuture.GetValueSync(); + + SourceFuture = + std::exchange(TargetFuture, TFuture()); + std::swap(SourceTest, TargetTest); + TargetTest.reset(); + + auto period = Config.GetMigrationSpec().GetMigrationPeriod(); + Scheduler->Schedule( + Timer->Now() + TDuration::Seconds(period), + [weakPtr = weak_from_this()] + { + if (auto self = weakPtr.lock()) { + self->Events.Enqueue(TStartTarget()); + } + }); + return true; + } + + bool HandleTestFinished(const TTestFinished& cmd) + { + STORAGE_INFO( + "%s test finished %s", + MakeTestTag().c_str(), + cmd.Id.c_str()); + + if (SourceTest && SourceTest->GetTestId() == cmd.Id) { + return false; + } + + if (TargetTest && TargetTest->GetTestId() == cmd.Id) { + return false; + } + + return true; + } + + bool HandleStateChange(TLoadTestControllerCommand cmd) + { + return std::visit( + TOverloaded{ + [this](const TStartSource& cmd) + { return HandleStartSource(cmd); }, + [this](const TStartTarget& cmd) + { return HandleStartTarget(cmd); }, + [this](const TFinishMigration& cmd) + { return HandleFinishMigration(cmd); }, + [this](const TTestFinished& cmd) + { + return HandleTestFinished(cmd); + }}, + cmd); + } + + void SetupTest() + { + Client->Start(); + + MaxDuration = TDuration::Seconds(Config.GetTestDuration()); + + if (Config.HasFileSystemId() && Config.HasCreateFileStoreRequest()) { + ythrow yexception() << MakeTestTag() + << " config should have either existing " + "filesystem id or request to create one"; + } + + if (Config.HasCreateFileStoreRequest()) { + auto request = std::make_shared( + Config.GetCreateFileStoreRequest()); + + STORAGE_INFO( + "%s create filestore: %s", + MakeTestTag().c_str(), + DumpMessage(*request).c_str()); + + TCallContextPtr ctx = MakeIntrusive(); + auto result = Client->CreateFileStore(ctx, request); + WaitForCompletion(GetRequestName(*request), result); + } + } + + void CleanupTest() + { + if (SourceTest) { + auto result = SendTeardownTest(SourceTest).GetValueSync(); + if (!result) { + STORAGE_ERROR( + "%s failed to teardown source client", + MakeTestTag().c_str()); + } + } + + if (TargetTest) { + auto result = SendTeardownTest(TargetTest).GetValueSync(); + if (!result) { + STORAGE_ERROR( + "%s failed to teardown source client", + MakeTestTag().c_str()); + } + TargetFuture.GetValueSync(); + } + + if (Config.HasCreateFileStoreRequest() && !Config.GetKeepFileStore()) { + auto filesystemId = + Config.GetCreateFileStoreRequest().GetFileSystemId(); + + STORAGE_INFO( + "%s destroy fs %s", + MakeTestTag().c_str(), + filesystemId.c_str()); + + auto request = std::make_shared(); + request->SetFileSystemId(filesystemId); + + TCallContextPtr ctx = MakeIntrusive(); + auto result = Client->DestroyFileStore(ctx, request); + WaitForCompletion(GetRequestName(*request), result); + } + + Client->Stop(); + + auto result = Result; + result.SetValue(SourceFuture.GetValueSync()); + } + + void* ThreadProc() override + { + StartTs = TInstant::Now(); + NCloud::SetCurrentThreadName(MakeTestTag()); + + SetupTest(); + + while (!ShouldStop()) { + try { + Events.Wait(); + bool cont = true; + while (auto maybeCmd = Events.Dequeue()) { + auto& cmd = *maybeCmd; + cont = HandleStateChange(cmd); + if (!cont) { + break; + } + } + if (!cont) { + break; + } + } catch (...) { + STORAGE_ERROR( + "%s test has failed: %s", + MakeTestTag().c_str(), + CurrentExceptionMessage().c_str()); + + Result.SetException(std::current_exception()); + } + } + + STORAGE_INFO("test is going to finish"); + + CleanupTest(); + + return nullptr; + } + +private: + bool ShouldStop() const + { + return AtomicGet(Ctx.ShouldStop) || + (MaxDuration && TInstant::Now() - StartTs >= MaxDuration) || + (TargetFuture.HasValue() || SourceFuture.HasValue()); + } + + const TString& MakeTestTag() const + { + return TestTag; + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +TString MakeTestTag(const TString& name) +{ + return Sprintf("[%s]", name.c_str()); +} + +ITestPtr CreateLoadTest( + const TAppContext& ctx, + const NProto::TLoadTest& config, + ITimerPtr timer, + ISchedulerPtr scheduler, + ILoggingServicePtr logging, + IClientFactoryPtr clientFactory) +{ + return std::make_shared( + ctx, + config, + std::move(timer), + std::move(scheduler), + std::move(logging), + std::move(clientFactory)); +} + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/test.h b/cloud/filestore/tools/testing/replay/lib/test.h new file mode 100644 index 0000000000..e4f9a7fcaf --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/test.h @@ -0,0 +1,38 @@ +#pragma once + +#include "public.h" + +#include + +#include +#include + +#include + +#include +#include + +namespace NCloud::NFileStore::NReplay { + +//////////////////////////////////////////////////////////////////////////////// + +struct ITest +{ + virtual ~ITest() = default; + + virtual NThreading::TFuture Run() = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TString MakeTestTag(const TString& name); + +ITestPtr CreateLoadTest( + const TAppContext& ctx, + const NProto::TLoadTest& config, + ITimerPtr timer, + ISchedulerPtr scheduler, + ILoggingServicePtr logging, + IClientFactoryPtr clientFactory); + +} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/ya.make b/cloud/filestore/tools/testing/replay/lib/ya.make new file mode 100644 index 0000000000..5851626fdc --- /dev/null +++ b/cloud/filestore/tools/testing/replay/lib/ya.make @@ -0,0 +1,36 @@ +LIBRARY(filestore-testing-replay-lib) + +SRCS( + client.cpp + context.h + executor.cpp + #request_data.cpp + #request_index.cpp + request_replay.cpp + test.cpp +) + +PEERDIR( + cloud/filestore/config + cloud/filestore/public/api/protos + + cloud/filestore/libs/client + cloud/filestore/libs/service + + cloud/storage/core/libs/common + cloud/storage/core/libs/diagnostics + + cloud/filestore/tools/testing/replay/protos + + library/cpp/deprecated/atomic + + library/cpp/eventlog + cloud/filestore/libs/diagnostics/events + cloud/filestore/tools/analytics/libs/event-log + library/cpp/testing/unittest + + library/cpp/aio +) + + +END() diff --git a/cloud/filestore/tools/testing/replay/protos/replay.proto b/cloud/filestore/tools/testing/replay/protos/replay.proto new file mode 100644 index 0000000000..2db7257abb --- /dev/null +++ b/cloud/filestore/tools/testing/replay/protos/replay.proto @@ -0,0 +1,125 @@ +syntax = "proto3"; + +package NCloud.NFileStore.NProto; + +import "cloud/filestore/public/api/protos/fs.proto"; + +//////////////////////////////////////////////////////////////////////////////// + + +enum EAction +{ + // INDEX + ACTION_CREATE_NODE = 0; + ACTION_RENAME_NODE = 1; + ACTION_REMOVE_NODE = 2; + ACTION_CREATE_HANDLE = 3; + ACTION_DESTROY_HANDLE = 4; + // FIXME: should allocate data before locking + ACTION_ACQUIRE_LOCK = 5; + ACTION_RELEASE_LOCK = 6; + ACTION_GET_NODE_ATTR = 7; + ACTION_ACCESS_NODE = 8; + + // DATA + ACTION_WRITE = 101; + ACTION_READ = 102; +}; + + +message TReplaySpec +{ + string FileName = 1; + //bool ValidationEnabled = 6; + //bool Sequential = 7; + string ReplayRoot = 8; + bool NoRead = 9; + bool NoWrite = 10; + bool Prepare = 11; // Only write files needed for reading + double TimeScale = 12; + bool WriteRandom = 13; + bool WriteEmpty = 14; +} + +message TMigrationSpec +{ + uint32 MigrationPeriod = 1; + uint32 StateTransferDelay = 2; +} + +message TLoadTest +{ + string Name = 1; + + oneof FileStore + { + // Existing filestore name + string FileSystemId = 2; + + // Create a new one + TCreateFileStoreRequest CreateFileStoreRequest = 3; + } + + oneof Specs + { + //TIndexLoadSpec IndexLoadSpec = 4; + //TDataLoadSpec DataLoadSpec = 5; + TReplaySpec ReplaySpec = 17; + } + + uint32 IODepth = 10; + uint32 RequestsCount = 11; + uint32 TestDuration = 12; + uint32 SessionPingTimeout = 13; + uint32 SessionRetryTimeout = 14; + + TMigrationSpec MigrationSpec = 15; + + // disables destruction of the filestore created via CreateFileStoreRequest + bool KeepFileStore = 16; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TLatency +{ + uint64 P50 = 1; + uint64 P95 = 2; + uint64 P90 = 3; + uint64 P99 = 4; + uint64 P999 = 5; + uint64 Min = 6; + uint64 Max = 7; + double Mean = 8; + double StdDeviation = 9; +} + +message TTestStats +{ + message TStats + { + string Action = 1; + uint64 Count = 2; + TLatency Latency = 3; + } + + string Name = 1; + bool Success = 2; + + repeated TStats Stats = 3; +} + +//////////////////////////////////////////////////////////////////////////////// + +message TTestGraph +{ + message TTest + { + oneof Test + { + TLoadTest LoadTest = 1; + } + } + + repeated TTest Tests = 1; +} diff --git a/cloud/filestore/tools/testing/replay/protos/ya.make b/cloud/filestore/tools/testing/replay/protos/ya.make new file mode 100644 index 0000000000..65814148f7 --- /dev/null +++ b/cloud/filestore/tools/testing/replay/protos/ya.make @@ -0,0 +1,13 @@ +PROTO_LIBRARY(filestore-testing-replay-protos) + +ONLY_TAGS(CPP_PROTO) + +PEERDIR( + cloud/filestore/public/api/protos +) + +SRCS( + replay.proto +) + +END() diff --git a/cloud/filestore/tools/testing/replay/ya.make b/cloud/filestore/tools/testing/replay/ya.make new file mode 100644 index 0000000000..908f2c16ef --- /dev/null +++ b/cloud/filestore/tools/testing/replay/ya.make @@ -0,0 +1,5 @@ +RECURSE( + bin + lib + protos +) From f79ae3bb2e01c2cf2890071aa5c5774194f7050b Mon Sep 17 00:00:00 2001 From: proller Date: Mon, 23 Sep 2024 17:08:43 +0000 Subject: [PATCH 03/44] clean --- .../tools/testing/replay/lib/request.h | 5 + .../testing/replay/lib/request_replay.cpp | 1648 +++-------------- .../tools/testing/replay/lib/test.cpp | 21 +- 3 files changed, 275 insertions(+), 1399 deletions(-) diff --git a/cloud/filestore/tools/testing/replay/lib/request.h b/cloud/filestore/tools/testing/replay/lib/request.h index a2ad107e58..ce99768233 100644 --- a/cloud/filestore/tools/testing/replay/lib/request.h +++ b/cloud/filestore/tools/testing/replay/lib/request.h @@ -47,6 +47,11 @@ struct IRequestGenerator virtual bool HasNextRequest() = 0; // virtual TInstant NextRequestAt() = 0; virtual NThreading::TFuture ExecuteNextRequest() = 0; + + virtual bool InstantProcessQueue() + { + return false; + } }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay.cpp b/cloud/filestore/tools/testing/replay/lib/request_replay.cpp index c0379083ce..e9c28f3c64 100644 --- a/cloud/filestore/tools/testing/replay/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/replay/lib/request_replay.cpp @@ -5,17 +5,12 @@ create file/dir modes create handle modes (now rw) compare log and actual result ( S_OK E_FS_NOENT ...) - - - - */ #include "request.h" #include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" #include "cloud/filestore/tools/analytics/libs/event-log/dump.h" -// #include "dump.h" #include "library/cpp/aio/aio.h" #include "library/cpp/eventlog/iterator.h" #include "library/cpp/testing/unittest/registar.h" @@ -51,83 +46,12 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -const ui64 SEGMENT_SIZE = 128; - -struct TSegment -{ - ui64 Handle = 0; - ui64 LastWriteRequestId = 0; -}; - -static_assert(sizeof(TSegment) <= SEGMENT_SIZE); -/* -bool Compare(const TSegment& expected, const TSegment& actual, TString* message) -{ - TStringBuilder sb; - - if (expected.Handle != actual.Handle) { - sb << "expected.Handle != actual.Handle: " << expected.Handle - << " != " << actual.Handle; - } - - if (expected.LastWriteRequestId != actual.LastWriteRequestId) { - if (sb.Size()) { - sb << ", "; - } - - sb << "expected.LastWriteRequestId != actual.LastWriteRequestId: " - << expected.LastWriteRequestId - << " != " << actual.LastWriteRequestId; - } - - *message = sb; - - return sb.Empty(); -} -*/ -struct TSegments - : TVector - , TAtomicRefCount -{ -}; - -using TSegmentsPtr = TIntrusivePtr; - -//////////////////////////////////////////////////////////////////////////////// - -struct THandleInfo -{ - TString Name; - ui64 Handle = 0; - ui64 Size = 0; - TSegmentsPtr Segments; - ui64 LastSlot = 0; - - THandleInfo() = default; - - THandleInfo( - TString name, - ui64 handle, - ui64 size, - TSegmentsPtr segments) noexcept - : Name(std::move(name)) - , Handle(handle) - , Size(size) - , Segments(std::move(segments)) - {} -}; - -//////////////////////////////////////////////////////////////////////////////// - class TReplayRequestGenerator final : public IRequestGenerator , public std::enable_shared_from_this { private: - // static constexpr ui32 DefaultBlockSize = 4_KB; - const NProto::TReplaySpec Spec; - // const TString FileSystemId; const NProto::THeaders Headers; @@ -136,35 +60,22 @@ class TReplayRequestGenerator final ISessionPtr Session; TVector> Actions; - // ui64 TotalRate = 0; TMutex StateLock; - ui64 LastWriteRequestId = 0; using THandleLog = ui64; using THandleLocal = ui64; using TNodeLog = ui64; using TNodeLocal = ui64; - TVector IncompleteHandleInfos; - TVector HandleInfos; - THashMap HandleInfosh; - THashMap HandlesLogToActual; - - // ui64 ReadBytes = DefaultBlockSize; - // ui64 WriteBytes = DefaultBlockSize; - // double AppendProbability = 1; - // ui64 BlockSize = DefaultBlockSize; ui64 InitialFileSize = 0; std::atomic LastRequestId = 0; - // NEventLog::IIterator * THolder EventlogIterator; TConstEventPtr EventPtr; int EventMessageNumber = 0; - NProto::TProfileLogRecord* messagep{}; - // TString FileSystemId; + NProto::TProfileLogRecord* EventLogMessagePtr{}; static constexpr ui32 LockLength = 4096; @@ -184,28 +95,20 @@ class TReplayRequestGenerator final ui64 Handle = 0; }; - THashMap StagedNodes; - THashMap Nodes; - // THashMap NodesLogToActual; - THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; - THashMap NodePath{{RootNodeId, "/"}}; - - //THashMap FileHandles; - THashMap Handles; - THashSet Locks; THashSet StagedLocks; + THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; + THashMap NodePath{{RootNodeId, "/"}}; + THashMap KnownLogNodes; + + THashMap HandlesLogToActual; + THashMap OpenHandles; + NAsyncIO::TAsyncIOService AsyncIO; - // TString ReplayRoot; ui64 TimestampMcs{}; - // ui64 DurationMcs{}; TInstant Started; - // THashMap OpenFiles; - // THashMap OpenHandles; - THashMap OpenHandles; - THashMap KnownLogNodes; public: TReplayRequestGenerator( @@ -221,90 +124,13 @@ class TReplayRequestGenerator final { Log = logging->CreateLog(Headers.GetClientId()); - // DUMP(spec); - - // ReplayRoot = spec.GetReplayRoot(); - AsyncIO.Start(); - /* - if (auto size = Spec.GetBlockSize()) { - BlockSize = size; - Y_ENSURE(BlockSize % DefaultBlockSize == 0); - } - if (auto bytes = Spec.GetReadBytes()) { - ReadBytes = bytes; - } - if (auto bytes = Spec.GetWriteBytes()) { - WriteBytes = bytes; - } - if (auto p = Spec.GetAppendPercentage()) { - AppendProbability = p / 100.; - } - - InitialFileSize = Spec.GetInitialFileSize(); - - if (Spec.GetValidationEnabled()) { - Y_ENSURE( - InitialFileSize % SEGMENT_SIZE == 0, - Sprintf( - "InitialFileSize (%lu) %% SEGMENT_SIZE (%lu) != 0", - InitialFileSize, - SEGMENT_SIZE - ) - ); - - Y_ENSURE( - WriteBytes % SEGMENT_SIZE == 0, - Sprintf( - "WriteBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", - WriteBytes, - SEGMENT_SIZE - ) - ); - - Y_ENSURE( - ReadBytes % SEGMENT_SIZE == 0, - Sprintf( - "ReadBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", - ReadBytes, - SEGMENT_SIZE - ) - ); - } - */ - - /* - for (const auto& action: Spec.GetActions()) { - Y_ENSURE(action.GetRate() > 0, "please specify positive - action rate"); - - TotalRate += action.GetRate(); - Actions.emplace_back(std::make_pair(TotalRate, - action.GetAction())); - } - - Y_ENSURE(!Actions.empty(), "please specify at least one action - for the test spec"); - */ - NEventLog::TOptions options; - // DUMP(Spec); - // Cerr << "SPEC=" << Spec; options.FileName = Spec.GetFileName(); - options.SetForceStrongOrdering(true); - // options.ForceStreamMode - // THolder it = CreateIterator(options, fac); - // DUMP(options.FileName); - // THolder it + options.SetForceStrongOrdering(true); // need this? EventlogIterator = CreateIterator(options); - // EventPtr = EventlogIterator->Next(); - // DUMP("opened"); - // NEventLog::IIterator* i - // EventlogIterator = it.Get(); - // DUMP((long)i); if (!Spec.GetReplayRoot().empty()) { - // auto fspath = TFsPath(Spec.GetReplayRoot()).MkDirs(); } } @@ -314,6 +140,11 @@ class TReplayRequestGenerator final AsyncIO.Stop(); } + bool InstantProcessQueue() override + { + return true; + } + TNodeLocal NodeIdMapped(const TNodeLog id) { if (const auto it = NodesLogToLocal.find(id); @@ -322,9 +153,8 @@ class TReplayRequestGenerator final return it->second; } - STORAGE_INFO( + STORAGE_DEBUG( "node not found " << id << " map size=" << NodesLogToLocal.size()); - // DUMP("but known=", KnownLogNodes.contains(id)); return 0; } @@ -335,7 +165,7 @@ class TReplayRequestGenerator final { return it->second; } - STORAGE_INFO( + STORAGE_DEBUG( "handle not found " << id << " map size=" << HandlesLogToActual.size()); return 0; @@ -349,310 +179,139 @@ class TReplayRequestGenerator final void Advance() { EventPtr = EventlogIterator->Next(); - // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); if (!EventPtr) { return; } - // if (EventPtr) { - // DUMP(EventPtr->GetName(),EventPtr->Class,EventPtr->FrameId); - //,EventPtr->ToString() - // const NProto::TProfileLogRecord* - messagep = const_cast( + EventLogMessagePtr = const_cast( dynamic_cast( EventPtr->GetProto())); - if (!messagep) { - // DUMP("nomesssss"); + if (!EventLogMessagePtr) { return; } - // DUMP(messagep->GetClassData(), messagep->GetFileSystemId()); if (FileSystemId.empty()) { - FileSystemId = TString{messagep->GetFileSystemId()}; + FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; } - EventMessageNumber = messagep->GetRequests().size(); + EventMessageNumber = EventLogMessagePtr->GetRequests().size(); } bool HasNextRequest() override { - // DUMP("hasnext?", !!EventPtr); if (!EventPtr) { - // EventPtr = EventlogIterator->Next(); Advance(); } - // DUMP("hasnext=", !!EventPtr); return !!EventPtr; } - /* TInstant NextRequestAt() override - { - return TInstant::Max(); - } - */ - NThreading::TFuture ExecuteNextRequest() override { - // DUMP("en"); - // auto *ev = EventlogIterator.Get(); - // EventPtr = EventlogIterator.Get(); -#if 0 - if (0) { - while (auto ev = EventlogIterator->Next()) { - // auto ev = EventlogIterator->Next(); - - DUMP(ev->GetName(), ev->ToString()); - - // event->Class; - - const auto* message = - dynamic_cast( - ev->GetProto()); - if (!message) { - DUMP("nomess"); - continue; - } - // const auto *message = ev->GetProto(); - // DUMP(message->AsJSON()); - // DUMP(message->GetTypeName()); - for (const auto& r: message->GetRequests()) { - DUMP(r.GetTypeName(), r.GetBlobsInfo(), r.GetRanges()); - DUMP(RequestName(r.GetRequestType())); - // TBasicString >&& () noexcept - // RequestName(r.GetRequestType()) = "GetNodeAttr"; - // {"TimestampMcs":1725548818281695,"DurationMcs":128,"RequestType":33,"ErrorCode":0,"NodeInfo":{"ParentNodeId":2395,"NodeName":"pre-receive.sample","Flags":0,"Mode":509,"NodeId":2402,"Handle":0,"Size":544}} - // DUMP(r.AsJSON()); - // Cerr << r.AsJSON() << "\n"; - DUMP(r.GetMetadata()); - } - } + if (!HasNextRequest()) { + return MakeFuture(TCompletedRequest{}); } -#endif - { - if (!HasNextRequest()) { - // DUMP("noreq"); - return MakeFuture(TCompletedRequest{}); - }; - // if (!EventPtr) {EventPtr = EventlogIterator->Next();} - - // NProto::TProfileLogRequestInfo request; - - // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); - // do { - for (; EventPtr; + for (; EventPtr; Advance()) { + if (!EventLogMessagePtr) { + continue; + } - // EventMessageNumber = 0, - // EventPtr = EventlogIterator->Next() - Advance()) - { - /* - if (!EventPtr) { - continue; - } - */ - // DUMP(!!EventPtr, (long)EventPtr.Get(), EventMessageNumber); - // if (EventPtr) { - /* - const NProto::TProfileLogRecord* messagep = - dynamic_cast( EventPtr->GetProto()); - */ - if (!messagep) { - // DUMP("nomess"); - continue; + STORAGE_DEBUG("Processing %d", EventMessageNumber); + for (; EventMessageNumber > 0;) { + auto request = + EventLogMessagePtr->GetRequests()[--EventMessageNumber]; + STORAGE_DEBUG("message json=" << request.AsJSON()); + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; } + const auto current = TInstant::Now(); + auto diff = current - Started; - /* const auto i = message->GetRequests().begin(); - for (const auto& r: message->GetRequests()) { - r.GetRequestType(); - } - */ - STORAGE_DEBUG("Processing %d", EventMessageNumber); - // DUMP(EventMessageNumber, messagep->GetRequests().size()); - /* - if (EventMessageNumber >= - messagep->GetRequests().size()) { DUMP("nonextreq"); - continue; - } - for (; EventMessageNumber < - messagep->GetRequests().size();) { auto request = - messagep->GetRequests()[EventMessageNumber++]; - */ - for (; EventMessageNumber > 0;) { - auto request = - messagep->GetRequests()[--EventMessageNumber]; - // TODO check ranges - // DUMP(request.AsJSON()); - // Cerr << request.AsJSON() << "\n"; - STORAGE_DEBUG("message json=" << request.AsJSON()); - // "TimestampMcs":1725981613498599,"DurationMcs":2101, - // TODO WTF - // 2024-09-10T14:52:07.145177Z nfs ListNodes - // 0.000385s S_OK {node_id=1, size=0} - // 2024-09-10T14:52:07.146009Z nfs ListNodes - // 0.000128s S_OK {node_id=1, size=0} - // 1970-01-01T00:14:52.767000Z nfs PingSession - // 0.000328s S_OK {no_info} - // 1970-01-01T00:14:53.768000Z nfs PingSession - // 0.000279s S_OK {no_info} - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - // DUMP(request.GetTimestampMcs(),TimestampMcs,timediff); - TimestampMcs = request.GetTimestampMcs(); - - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - - if (timediff > diff.MicroSeconds()) { - auto slp = TDuration::MicroSeconds( - timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); + if (timediff > diff.MicroSeconds()) { + auto slp = + TDuration::MicroSeconds(timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); - Sleep(slp); // TODO: calculate correct timescale here - } - // DUMP(current, Started, diff.MicroSeconds()); - Started = current; + Sleep(slp); + } + Started = current; - // DurationMcs = - // request.GetDurationMcs(); + STORAGE_DEBUG( + "Processing message " + << EventMessageNumber + << " typename=" << request.GetTypeName() + << " type=" << request.GetRequestType() << " " + << RequestName(request.GetRequestType())); - STORAGE_DEBUG( - "Processing message " - << EventMessageNumber - << " typename=" << request.GetTypeName() - << " type=" << request.GetRequestType() << " " - << RequestName(request.GetRequestType())); - - { - const auto& action = request.GetRequestType(); - switch (static_cast(action)) { - case EFileStoreRequest:: - ReadData: // NProto::ACTION_READ: - return DoReadData(request); - case EFileStoreRequest:: - WriteData: // NProto::ACTION_WRITE: - return DoWrite(request); - // static_cast(requestType) - case EFileStoreRequest:: - CreateNode: // NProto::ACTION_CREATE_NODE: - return DoCreateNode(request); - case EFileStoreRequest:: - RenameNode: // NProto::ACTION_RENAME_NODE: - return DoRenameNode(request); - case EFileStoreRequest:: - UnlinkNode: // NProto::ACTION_REMOVE_NODE: - return DoUnlinkNode(request); - case EFileStoreRequest:: - CreateHandle: // NProto::ACTION_CREATE_HANDLE: - return DoCreateHandle(request); - case EFileStoreRequest:: - DestroyHandle: // NProto::ACTION_DESTROY_HANDLE: - return DoDestroyHandle(request); - case EFileStoreRequest:: - GetNodeAttr: // NProto::ACTION_GET_NODE_ATTR: - return DoGetNodeAttr(request); - case EFileStoreRequest:: - AcquireLock: // NProto::ACTION_ACQUIRE_LOCK: - return DoAcquireLock(); - case EFileStoreRequest:: - ReleaseLock: // NProto::ACTION_RELEASE_LOCK: - return DoReleaseLock(); - - case EFileStoreRequest:: - AccessNode: // NProto::ACTION_CREATE_HANDLE: - return DoAccessNode(request); - - case EFileStoreRequest::ReadBlob: - case EFileStoreRequest::WriteBlob: - case EFileStoreRequest::GenerateBlobIds: - case EFileStoreRequest::PingSession: - case EFileStoreRequest::Ping: - - continue; - - /* TODO: - 2024-09-19T23:54:21.789965Z :smoke DEBUG: - cloud/filestore/tools/testing/replay/lib/request_replay.cpp:475: - message - json={"TimestampMcs":1726503858215279,"DurationMcs":20865,"RequestType":10001,"ErrorCode":0,"BlobsInfo":[{"CommitId":4295169028,"Unique":1037938976620549,"Ranges":[{"NodeId":16064,"Offset":1179648,"Bytes":81920},{"NodeId":16065,"Offset":1179648,"Bytes":81920},{"NodeId":16066,"Offset":1179648,"Bytes":77824}]},{"CommitId":4295169028,"Unique":1636073302130950,"Ranges":[{"NodeId":16050,"Offset":0,"Bytes":262144},{"NodeId":16051,"Offset":0,"Bytes":16384},{"NodeId":16052,"Offset":0,"Bytes":49152},{"NodeId":16060,"Offset":0,"Bytes":20480},{"NodeId":16061,"Offset":0,"Bytes":32768}]},{"CommitId":4295169028,"Unique":1125899906843139,"Ranges":[{"NodeId":16050,"Offset":262144,"Bytes":262144}]},{"CommitId":4295169028,"Unique":211106232533764,"Ranges":[{"NodeId":16040,"Offset":0,"Bytes":49152}]},{"CommitId":4295169028,"Unique":1125899906843653,"Ranges":[{"NodeId":16050,"Offset":524288,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906843910,"Ranges":[{"NodeId":16050,"Offset":786432,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1952732650931715,"Ranges":[{"NodeId":16050,"Offset":1048576,"Bytes":262144},{"NodeId":16062,"Offset":1179648,"Bytes":114688},{"NodeId":16063,"Offset":1179648,"Bytes":77824}]},{"CommitId":4295169028,"Unique":1125899906844420,"Ranges":[{"NodeId":16050,"Offset":1310720,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906844677,"Ranges":[{"NodeId":16050,"Offset":1572864,"Bytes":262144}]},{"CommitId":4295169028,"Unique":52776558135558,"Ranges":[{"NodeId":16041,"Offset":1822720,"Bytes":12288}]},{"CommitId":4295169028,"Unique":1125899906845187,"Ranges":[{"NodeId":16050,"Offset":1835008,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845444,"Ranges":[{"NodeId":16041,"Offset":1835008,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845701,"Ranges":[{"NodeId":16050,"Offset":2097152,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906845958,"Ranges":[{"NodeId":16041,"Offset":2097152,"Bytes":262144}]},{"CommitId":4295169028,"Unique":1125899906846211,"Ranges":[{"NodeId":16050,"Offset":2359296,"Bytes":262144}]},{"CommitId":4295169028,"Unique":439804651114244,"Ranges":[{"NodeId":16041,"Offset":2359296,"Bytes":102400}]},{"CommitId":4295169028,"Unique":1125899906846725,"Ranges":[{"NodeId":16050,"Offset":2621440,"Bytes":262144}]},{"CommitId":4295169028,"Unique":175921860448518,"Ranges":[{"NodeId":16050,"Offset":2883584,"Bytes":40960}]},{"CommitId":4295169028,"Unique":123145302315523,"Ranges":[{"NodeId":16067,"Offset":0,"Bytes":28672}]}]} - 2024-09-19T23:54:21.789970Z :smoke DEBUG: - cloud/filestore/tools/testing/replay/lib/request_replay.cpp:517: - Processing message 4294 - typename=NCloud.NFileStore.NProto.TProfileLogRequestInfo - type=10001 Flush - */ - - // listnodes ->. noeid -> createdir - - // 9: CreateSession - - default: - STORAGE_INFO( - "Uninmplemented action=" - << action << " " - << RequestName(request.GetRequestType())); - // Y_ABORT("unexpected action: %u", - // (ui32)action); - continue; - } + { + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest::ReadData: + return DoReadData(request); + case EFileStoreRequest::WriteData: + return DoWrite(request); + case EFileStoreRequest::CreateNode: + return DoCreateNode(request); + case EFileStoreRequest::RenameNode: + return DoRenameNode(request); + case EFileStoreRequest::UnlinkNode: + return DoUnlinkNode(request); + case EFileStoreRequest::CreateHandle: + return DoCreateHandle(request); + case EFileStoreRequest::DestroyHandle: + return DoDestroyHandle(request); + case EFileStoreRequest::GetNodeAttr: + return DoGetNodeAttr(request); + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(); + + case EFileStoreRequest::AccessNode: + return DoAccessNode(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + continue; + default: + STORAGE_INFO( + "Uninmplemented action=" + << action << " " + << RequestName(request.GetRequestType())); + continue; } } - - // break; - - // message.get } - STORAGE_DEBUG( - "Finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); - //} while (EventMessageNumber = 0, EventPtr = - // EventlogIterator->Next()); - - /* - // const auto& action = PeekNextAction(); - */ - /* - return MakeFuture(TCompletedRequest{ - NProto::EAction::ACTION_READ, - TInstant::Now(), - NProto::TError{}}); - // return DoRead(); - */ - - return MakeFuture(TCompletedRequest(true)); } + STORAGE_DEBUG( + "Finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); + + return MakeFuture(TCompletedRequest(true)); } private: - /* - NProto::EAction PeekNextAction() - { - auto number = RandomNumber(TotalRate); - auto it = LowerBound( - Actions.begin(), - Actions.end(), - number, - [](const auto& pair, ui64 b) { return pair.first < b; }); - - Y_ABORT_UNLESS(it != Actions.end()); - return it->second; - } - */ - TFuture DoAccessNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} + + auto started = TInstant::Now(); + if (Spec.GetNoRead()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } - // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} TGuard guard(StateLock); - auto started = TInstant::Now(); if (UseFs()) { const auto node = NodeIdMapped(r.GetNodeInfo().GetNodeId()); @@ -674,132 +333,78 @@ class TReplayRequestGenerator final return MakeFuture( TCompletedRequest{NProto::ACTION_ACCESS_NODE, started, {}}); } - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); } // Recursive, no infinity loop check TNodeLocal CreateDirIfMissingByNodeLog(TNodeLog nodeIdLog) { if (const auto& nodeIdLocal = NodeIdMapped(nodeIdLog)) { - // DUMP("Already created", nodeIdLog, "->", nodeIdLocal); return nodeIdLocal; } - // DUMP(nodeIdLog); - const auto& it = KnownLogNodes.find(nodeIdLog); if (it == KnownLogNodes.end()) { - // DUMP("not known", NodePath, "but", NodeIdMapped(nodeIdLog)); - // maybe create in tmpdir return 0; } - /* - DUMP( - "WANACREEEEE", - nodeIdLog, - it->second.Name, - it->second.ParentLog, - NodeIdMapped(it->second.ParentLog)); - */ auto parent = NodeIdMapped(it->second.ParentLog); if (!parent && it->second.ParentLog && nodeIdLog != it->second.ParentLog) { - // NodeIdMapped() parent = CreateDirIfMissingByNodeLog(it->second.ParentLog); - // DUMP("create miss parent", it->second.ParentLog, " => ", parent); } - // if (!parent) { auto parentPath = PathByNode(NodeIdMapped(it->second.ParentLog)); - /* - DUMP( - "try1", - parentPath, - parent, - it->second.ParentLog, - parentPath, - NodeIdMapped(it->second.ParentLog)); - */ - // KnownLogNodes.contains() if (parentPath.empty() && parent) { parentPath = PathByNode(parent); - // DUMP("try2", parent, parentPath); } if (parentPath.empty()) { parentPath = "/__lost__/"; } - // DUMP("tryR", parent, it->second.ParentLog, parentPath); - - // const auto name = (parent ? PathByNode(parent) : "__lost__/") + - // ToString(logNodeId) + "/"; const auto name = parentPath + - // ToString(logNodeId) + "/"; const auto name = parentPath + (it->second.Name.empty() ? ToString(nodeIdLog) : it->second.Name) + "/"; - /* - DUMP( - "namegen", - parent, - PathByNode(parent), - it->second.Parent, - PathByNode(it->second.Parent), - parentPath, - logNodeId); - */ - const auto nodeId = Mkdir(Spec.GetReplayRoot() + name); + const auto nodeId = + MakeDirectoryRecursive(Spec.GetReplayRoot() + name); NodePath[nodeId] = name; - // DUMP("savepath", nodeId, name); NodesLogToLocal[nodeIdLog] = nodeId; - // DUMP("created lost", nodeid, name); return nodeId; } - /* - const auto name = PathByNode(parent) + "/" + it->second.Name; - const auto nodeid = Mkdir(Spec.GetReplayRoot() + "/" + name); - NodePath[nodeid] = name; - - return nodeid; - */ } TFuture DoCreateHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { - // message // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} + // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} + // nfs CreateHandle 0.004161s S_OK {parent_node_id=65, + // node_name=ini, flags=14, mode=436, node_id=66, + // handle=11024287581389312, size=0} + TGuard guard(StateLock); auto started = TInstant::Now(); if (UseFs()) { - /// DUMP("hhhhhhh================================================"); TString relativePathName; if (r.GetNodeInfo().GetNodeId()) { if (auto path = PathByNode(r.GetNodeInfo().GetNodeId())) { relativePathName = path; } - // DUMP("open by nodeid", relativePathName); - // TODO: maybe open by fhandle } if (relativePathName.empty()) { auto parentNode = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - /* - DUMP( - "mapped", - parentNode, - r.GetNodeInfo().GetParentNodeId(), - PathByNode(parentNode)); - */ if (!parentNode) { parentNode = NodeIdMapped( KnownLogNodes[r.GetNodeInfo().GetParentNodeId()] .ParentLog); - // DUMP("known", parentNode); } if (!parentNode && r.GetNodeInfo().GetParentNodeId() != @@ -807,7 +412,6 @@ class TReplayRequestGenerator final { parentNode = CreateDirIfMissingByNodeLog( r.GetNodeInfo().GetParentNodeId()); - // DUMP("create", parentNode); } if (!parentNode) { @@ -815,118 +419,47 @@ class TReplayRequestGenerator final "create handle fail :" << r.GetNodeInfo().GetHandle() << " no parent=" << r.GetNodeInfo().GetParentNodeId()); - // DUMP( - // "noparent",r.GetNodeInfo().GetParentNodeId(),r.GetNodeInfo().GetHandle()); - // DUMP(NodesLogToActual); return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, started, MakeError(E_FAIL, "cancelled")}); } - auto nodename = r.GetNodeInfo().GetNodeName(); - if (nodename.empty() && r.GetNodeInfo().GetNodeId() != + auto nodeName = r.GetNodeInfo().GetNodeName(); + if (nodeName.empty() && r.GetNodeInfo().GetNodeId() != r.GetNodeInfo().GetParentNodeId()) { - nodename = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; - // DUMP("node name known", nodename); + nodeName = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; } auto parentpath = PathByNode(parentNode); - if (nodename.empty() && + if (nodeName.empty() && IsDir(Spec.GetReplayRoot() + parentpath)) { - nodename = + nodeName = KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name; } - // DUMP("can add - // name",KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name); - // DUMP(parentpath,nodename,NFs::Exists(Spec.GetReplayRoot() + - // parentpath)); - relativePathName = - //"/" + - // PathByNode(parentNode) + - // r.GetNodeInfo().GetNewNodeName(); - parentpath + nodename; - /* - DUMP( - "fname", - relativePathName, - parentNode, - parentpath, - PathByNode(parentNode), - nodename); - */ + relativePathName = parentpath + nodeName; } - // PathByNode(r.GetNodeInfo().GetNodeId())+ - // r.GetNodeInfo().GetNewNodeName(); STORAGE_DEBUG( - "open " - << relativePathName // << " in " << parentNode << " <- " - << " handle=" << r.GetNodeInfo().GetHandle() - << " flags=" << r.GetNodeInfo().GetFlags() - << " mode=" << r.GetNodeInfo().GetMode() - << " node=" << r.GetNodeInfo().GetNodeId() - - ) - - ; + "open " << relativePathName + << " handle=" << r.GetNodeInfo().GetHandle() + << " flags=" << r.GetNodeInfo().GetFlags() + << " mode=" << r.GetNodeInfo().GetMode() + << " node=" << r.GetNodeInfo().GetNodeId()); - // ui64 id = 0; - // DUMP("file", fname); - - /* - if (OpenFiles.contains(fname)) { - DUMP( - "already opened", - fname, - OpenFiles[fname], - r.GetNodeInfo().GetHandle()); - // return MakeFuture(TCompletedRequest{}); - } - */ - // DUMP(NFs::Exists(Spec.GetReplayRoot() + relativePathName)); - // TFileHandle - TFile FileHandle( + TFile fileHandle( Spec.GetReplayRoot() + relativePathName, - // static_cast(r.GetNodeInfo().GetMode()) - OpenAlways | RdWr // RdOnly - - ); - - if (!FileHandle.IsOpen()) { - TFile test{ - Spec.GetReplayRoot() + relativePathName, - OpenAlways | RdWr}; - /* - DUMP( - " seconddtest", - test.IsOpen(), - test.GetName(), - test.GetLength(), - test.GetPosition(), - test.GetHandle()); - */ - } - - /* - DUMP( - FileHandle.IsOpen(), - FileHandle.GetPosition(), - FileHandle.GetLength()); - */ + OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - if (!FileHandle.IsOpen()) { - // DUMP("handle open fail", r.GetNodeInfo().GetHandle()); + if (!fileHandle.IsOpen()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, started, MakeError(E_FAIL, "fail")}); } - // const auto fh = (FHANDLE)FileHandle; - const auto fh = FileHandle.GetHandle(); - // DUMP(fh); + const auto fh = fileHandle.GetHandle(); if (!fh) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, @@ -934,99 +467,30 @@ class TReplayRequestGenerator final MakeError(E_FAIL, "no filehandle")}); } - // if (fh) { - - /* - auto fh = ::open( - fname.c_str(), - r.GetNodeInfo().GetFlags() | O_CREAT, - r.GetNodeInfo().GetMode()); - DUMP(fh); - if (fh <= 0) { - DUMP("fail to create handle"); - return MakeFuture(TCompletedRequest{}); - } - */ - // OpenFiles[fname] = fh; - OpenHandles[fh] = std::move(FileHandle); - // TFileStat{}; - // const auto id = (FHANDLE)(fh); - // TFileStat stat{id}; - /* - TFileStat stat{fh}; - // GetStatByHandle(stat, id); - auto id = stat.INode; - - FileHandles[id] = std::move(fh); - */ - // DUMP(stat.INode, stat.IsDir(), stat.IsFile()); - - // DUMP(id, fh.GetLength(), fh.GetPosition()); - - // CreateIfMissing(PathByNode()) + OpenHandles[fh] = std::move(fileHandle); HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; - // DUMP("savehandle", r.GetNodeInfo().GetHandle(), fh); const auto inode = TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; if (r.GetNodeInfo().GetNodeId()) { NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; - // NodePath[id] = PathByNode(parentNode) - // +r.GetNodeInfo().GetNewNodeName() +(isDir ? "/" : ""); NodePath[inode] = relativePathName; - /* - DUMP( - "savepath2", - r.GetNodeInfo().GetNodeId(), - NodesLogToLocal[r.GetNodeInfo().GetNodeId()], - inode, - relativePathName); - */ } - // NodePath[id] = PathByNode(parentNode) - // +r.GetNodeInfo().GetNewNodeName() +(isDir ? "/" : ""); - // DUMP(NodePath[id], parentNode); STORAGE_DEBUG( "open " << fh << "<-" << r.GetNodeInfo().GetHandle() - << " known handles=" << HandlesLogToActual.size() << " opened=" << OpenHandles.size() - << " inode=" << inode) - - ; + << " inode=" << inode); - // DUMP(NodePath); - // DUMP("open ok", HandlesLogToActual); - // return {}; return MakeFuture( TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); } - /* - static const int flags = - ProtoFlag(NProto::TCreateHandleRequest::E_CREATE) | - ProtoFlag(NProto::TCreateHandleRequest::E_EXCLUSIVE) | - ProtoFlag(NProto::TCreateHandleRequest::E_READ) | - ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); - */ - /* - auto name = GenerateNodeName(); - */ auto request = CreateRequest(); - // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} - // nfs CreateHandle 0.004161s S_OK {parent_node_id=65, - // node_name=ini, flags=14, mode=436, node_id=66, - // handle=11024287581389312, size=0} - /* - request->SetNodeId(RootNodeId); - request->SetName(name); - request->SetFlags(flags); - */ auto name = r.GetNodeInfo().GetNodeName(); const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); if (!node) { - // DUMP("nop", r.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest{}); } request->SetNodeId(node); @@ -1034,8 +498,6 @@ class TReplayRequestGenerator final request->SetFlags(r.GetNodeInfo().GetFlags()); request->SetMode(r.GetNodeInfo().GetMode()); - Cerr << "crrate=" << *request << "\n"; - auto self = weak_from_this(); return Session->CreateHandle(CreateCallContext(), std::move(request)) .Apply( @@ -1065,20 +527,7 @@ class TReplayRequestGenerator final CheckResponse(response); auto handle = response.GetHandle(); - auto& infos = InitialFileSize ? IncompleteHandleInfos : HandleInfos; with_lock (StateLock) { - TSegmentsPtr segments; - - /* - if (Spec.GetValidationEnabled()) { - segments = new TSegments(); - segments->resize(Spec.GetInitialFileSize() - / SEGMENT_SIZE); - } - */ - - infos.emplace_back(name, handle, 0, std::move(segments)); - HandleInfosh[handle] = {name, handle, 0, std::move(segments)}; HandlesLogToActual[r.GetNodeInfo().GetHandle()] = handle; } @@ -1102,8 +551,12 @@ class TReplayRequestGenerator final } return setAttr.Apply( - [=, this](const TFuture& f) - { return HandleResizeAfterCreateHandle(f, name, started); }); + [=]([[maybe_unused]] const TFuture< + NProto::TSetNodeAttrResponse>& f) + { + return MakeFuture(TCompletedRequest{}); + // return HandleResizeAfterCreateHandle(f, name, started); + }); } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( @@ -1118,70 +571,8 @@ class TReplayRequestGenerator final } } - TCompletedRequest HandleResizeAfterCreateHandle( - const TFuture& future, - const TString& name, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - bool handleFound = false; - with_lock (StateLock) { - ui32 handleIdx = 0; - while (handleIdx < IncompleteHandleInfos.size()) { - auto& hinfo = IncompleteHandleInfos[handleIdx]; - if (hinfo.Name == name) { - handleFound = true; - - hinfo.Size = - Max(hinfo.Size, response.GetNode().GetSize()); - - STORAGE_INFO( - "updated file size for handle %lu and file %s" - ", size=%lu", - hinfo.Handle, - name.Quote().c_str(), - hinfo.Size); - - break; - } - - ++handleIdx; - } - - if (handleIdx < IncompleteHandleInfos.size()) { - DoSwap( - IncompleteHandleInfos[handleIdx], - IncompleteHandleInfos.back()); - - HandleInfos.push_back( - std::move(IncompleteHandleInfos.back())); - - IncompleteHandleInfos.pop_back(); - } - } - - if (!handleFound) { - STORAGE_WARN( - "handle for file %s not found", - name.Quote().c_str()); - } - - return {NProto::ACTION_CREATE_HANDLE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "set node attr handle for %s has failed: %s", - name.Quote().c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_HANDLE, started, error}; - } - } - static constexpr ui32 BlockSize = 4_KB; - std::shared_ptr Acalloc(ui64 dataSize) + static std::shared_ptr Acalloc(ui64 dataSize) { std::shared_ptr buffer = { static_cast(aligned_alloc(BlockSize, dataSize)), @@ -1196,15 +587,17 @@ class TReplayRequestGenerator final TFuture DoReadData( NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + const auto started = TInstant::Now(); if (Spec.GetNoRead()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto started = TInstant::Now(); if (UseFs()) { const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); - // DUMP(handle, r.GetRanges(0).GetHandle()); if (!handle) { STORAGE_WARN( "read: no handle " @@ -1216,75 +609,32 @@ class TReplayRequestGenerator final started, MakeError(E_FAIL, "cancelled")}); } - // auto bytes = r.GetRanges(0).GetBytes(); - // auto buffer = NUnitTest::RandomString(bytes); - - // auto fh = TFileHandle{static_cast(handle)}; auto& fh = OpenHandles[handle]; STORAGE_DEBUG( "Read from " << handle << " fh.len=" << fh.GetLength() << " fh.pos=" << fh.GetPosition()); - // AsyncIO.Write( (FHANDLE)(handle), buffer.data(), bytes, - // r.GetRanges(0).GetOffset()); auto buffer = Acalloc(r.GetRanges().cbegin()->GetBytes()); - //[[maybe_unused]] const auto reserved = fh.Reserve( r.GetRanges().cbegin()->GetOffset() + r.GetRanges().cbegin()->GetBytes()); - // DUMP("gorrread",reserved,r.GetRanges().cbegin()->GetOffset(),r.GetRanges().cbegin()->GetBytes()); TFileHandle FileHandle{fh.GetHandle()}; const auto future = AsyncIO.Read( - // fh, FileHandle, {}, r.GetRanges().cbegin()->GetBytes(), - r.GetRanges().cbegin()->GetOffset()) - - //.Apply([](const auto& v){DUMP("rrrres", v.GetValue());}) - ; + r.GetRanges().cbegin()->GetOffset()); FileHandle.Release(); - // fh.Release(); return future.Apply( - [started]([[maybe_unused]] const auto& future) mutable - { - // DUMP("writeresult", future.GetValue()); + [started]([[maybe_unused]] const auto& future) mutable { return TCompletedRequest(NProto::ACTION_READ, started, {}); }); - - // return MakeFuture(TCompletedRequest{NProto::ACTION_READ, started, - // {}}); } - // maybe before? - /* - if (HandleInfos.empty()) { - return DoCreateHandle(); - } - */ - // auto handleInfo = GetHandleInfo(); - /* - if (handleInfo.Size < ReadBytes) { - return DoWrite(handleInfo); - } - */ - - // return {}; - - /* - const ui64 slotOffset = PickSlot(handleInfo, ReadBytes); - const ui64 byteOffset = slotOffset * ReadBytes; - */ auto request = CreateRequest(); - /* - request->SetHandle(handleInfo.Handle); - request->SetOffset(byteOffset); - request->SetLength(ReadBytes); - */ - // ensure exists const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); if (!handle) { return MakeFuture(TCompletedRequest{}); @@ -1320,53 +670,17 @@ class TReplayRequestGenerator final TCompletedRequest HandleRead( const TFuture& future, - THandleInfo handleInfo, - TInstant started - //,ui64 byteOffset - ) + TInstant started) { try { auto response = future.GetValue(); CheckResponse(response); - - /* - const auto& buffer = response.GetBuffer(); - - // DUMP("read",buffer.Size(),TStringBuf(buffer, 0, - // std::min(10, buffer.size()))); - - ui64 segmentId = byteOffset / SEGMENT_SIZE; - for (ui64 offset = 0; offset < ReadBytes; offset += - SEGMENT_SIZE) { const TSegment* segment = reinterpret_cast(buffer.data() + offset); if - (Spec.GetValidationEnabled()) { - Y_ABORT_UNLESS(handleInfo.Segments); - - auto& segments = *handleInfo.Segments; - Y_ABORT_UNLESS(segmentId < segments.size()); - - TString message; - if (!Compare(segments[segmentId], *segment, - &message)) { throw TServiceError(E_FAIL) << Sprintf( "Validation - failed: %s", message.c_str()); - } - } - ++segmentId; - } - */ - - /* - with_lock (StateLock) { - HandleInfos.emplace_back(std::move(handleInfo)); - } - */ - return {NProto::ACTION_READ, started, response.GetError()}; } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( - "read for %s has failed: %s", - handleInfo.Name.Quote().c_str(), + "read for %s has failed: ", + // handleInfo.Name.Quote().c_str(), FormatError(error).c_str()); return {NProto::ACTION_READ, started, error}; @@ -1376,28 +690,30 @@ class TReplayRequestGenerator final TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) { TStringBuilder ret; - ret << start; + ret << "[\n" << start; while (ret.size() < bytes) { - ret << " . " << offset + ret.size(); + ret << " . " << ret.size() << " : " << offset + ret.size(); } return ret.substr(0, bytes); } TFuture DoWrite( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r - // THandleInfo handleInfo = {} - ) + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} + + const auto started = TInstant::Now(); if (Spec.GetNoWrite()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_WRITE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto started = TInstant::Now(); if (UseFs()) { const auto logHandle = r.GetRanges(0).GetHandle(); const auto handle = HandleIdMapped(logHandle); - // DUMP(handle, r.GetRanges(0).GetHandle()); if (!handle) { return MakeFuture(TCompletedRequest( NProto::ACTION_WRITE, @@ -1420,24 +736,17 @@ class TReplayRequestGenerator final buffer = MakeBuffer( bytes, offset, - TStringBuilder{} << "[ handle=" << logHandle + TStringBuilder{} << "handle=" << logHandle << " node=" << r.GetNodeInfo().GetNodeId() << " bytes=" << bytes << " offset=" << offset); } - // auto fh = TFileHandle{static_cast(handle)}; - auto& fh = OpenHandles[handle]; STORAGE_DEBUG( "Write to " << handle << " fh.length=" << fh.GetLength() << " fh.pos=" << fh.GetPosition()); - // AsyncIO.Write( (FHANDLE)(handle), buffer.data(), bytes, - // r.GetRanges(0).GetOffset()); - // const auto fut = AsyncIO.Write(fh, buffer.data(), bytes, - // r.GetRanges(0).GetOffset()).Apply([](const auto& v) { - // DUMP("wwwwwwres", v.GetValue()); }); // TODO TEST USE AFTER FREE on buffer TFileHandle FileHandle{fh.GetHandle()}; const auto writeFuture = AsyncIO.Write( @@ -1448,45 +757,12 @@ class TReplayRequestGenerator final offset); FileHandle.Release(); return writeFuture.Apply( - [started]([[maybe_unused]] const auto& future) mutable - { - // DUMP("writeresult", future.GetValue()); + [started]([[maybe_unused]] const auto& future) mutable { return TCompletedRequest(NProto::ACTION_WRITE, started, {}); }); - - // return MakeFuture(TCompletedRequest{}); } - /* - if (!handleInfo.Handle) { - if (HandleInfos.empty()) { - return DoCreateHandle(); - } - - handleInfo = GetHandleInfo(); - } - - const auto started = TInstant::Now(); - ui64 byteOffset = handleInfo.Size; - if (RandomNumber() < AppendProbability) { - handleInfo.Size += WriteBytes; - } else { - handleInfo.Size = Max(handleInfo.Size, WriteBytes); - const ui64 slotOffset = PickSlot(handleInfo, WriteBytes); - byteOffset = slotOffset * WriteBytes; - } - - auto request = CreateRequest(); - request->SetHandle(handleInfo.Handle); - request->SetOffset(byteOffset); - */ - // request->SetBuffer(); auto request = CreateRequest(); - /*if (HandlesLogToActual.contains(r.GetRanges(0).GetHandle())) { - DUMP("unknown handle", r.GetRanges(0).GetHandle()); - } - request->SetHandle(HandlesLogToActual[r.GetRanges(0).GetHandle()]); -*/ const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); if (!handle) { @@ -1494,54 +770,19 @@ class TReplayRequestGenerator final } request->SetHandle(handle); - // auto byteOffset = r.GetRanges(0).GetOffset(); request->SetOffset(r.GetRanges(0).GetOffset()); auto bytes = r.GetRanges(0).GetBytes(); - //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} - // request.set - // TString buffer(bytes, '\0'); - // NUnitTest::RandomString() - // GenerateRandomString - // GenerateRandomData auto buffer = NUnitTest::RandomString(bytes); *request->MutableBuffer() = std::move(buffer); - ++LastWriteRequestId; - /* - TString buffer(WriteBytes, '\0'); - ui64 segmentId = byteOffset / SEGMENT_SIZE; - for (ui64 offset = 0; offset < WriteBytes; offset += - SEGMENT_SIZE) { TSegment* segment = - reinterpret_cast(buffer.begin() + offset); - segment->Handle = handleInfo.Handle; - segment->LastWriteRequestId = LastWriteRequestId; - - if (Spec.GetValidationEnabled()) { - Y_ABORT_UNLESS(handleInfo.Segments); - - auto& segments = *handleInfo.Segments; - if (segments.size() <= segmentId) { - segments.emplace_back(); - } - - segments[segmentId] = *segment; - } - - ++segmentId; - } - *request->MutableBuffer() = std::move(buffer); - */ auto self = weak_from_this(); return Session->WriteData(CreateCallContext(), std::move(request)) .Apply( [=](const TFuture& future) { if (auto ptr = self.lock()) { - return ptr->HandleWrite( - future, - // handleInfo, - started); + return ptr->HandleWrite(future, started); } return TCompletedRequest{ @@ -1553,17 +794,12 @@ class TReplayRequestGenerator final TCompletedRequest HandleWrite( const TFuture& future, - // THandleInfo handleInfo, TInstant started) { try { auto response = future.GetValue(); CheckResponse(response); - with_lock (StateLock) { - // HandleInfos.emplace_back(std::move(handleInfo)); - } - return {NProto::ACTION_WRITE, started, response.GetError()}; } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); @@ -1585,210 +821,136 @@ class TReplayRequestGenerator final return {}; } - /* - void CreateIfMissing(TString path) - { - TFsPath(path).MkDirs(); - } - */ - - static TNodeLocal Mkdir(const TString& name) + static TNodeLocal MakeDirectoryRecursive(const TString& name) { - //[[maybe_unused]] const auto mkdires = NFs::MakeDirectoryRecursive(name); - // mkdir(fname.c_str(), r.GetNodeInfo().GetMode()); const auto inode = TFileStat{name}.INode; - // DUMP("mkdir", inode, name); return inode; } TFuture DoCreateNode( NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} + // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, + // new_node_name=home, mode=509, node_id=12526, size=0} + + const auto started = TInstant::Now(); if (Spec.GetNoWrite()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto started = TInstant::Now(); - /* - auto name = GenerateNodeName(); - StagedNodes[name] = {}; - */ if (UseFs()) { - // DUMP("========================================================"); - auto parentNode = NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); parentNode = CreateDirIfMissingByNodeLog( r.GetNodeInfo().GetNewParentNodeId()); - // DUMP(parentNode, PathByNode(parentNode)); - - auto fname = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + - r.GetNodeInfo().GetNewNodeName(); - // DUMP(Spec.GetReplayRoot(), fname); + auto fullName = Spec.GetReplayRoot() + "/" + + PathByNode(parentNode) + + r.GetNodeInfo().GetNewNodeName(); ui64 nodeid = 0; bool isDir = false; switch (r.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { - // DUMP("file", fname); - + // TODO: transform r.GetNodeInfo().GetMode() to correct open + // mode TFileHandle fh( - fname, - // static_cast(r.GetNodeInfo().GetMode()) - OpenAlways | RdWr // RdOnly - ); + fullName, + OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); fh.Reserve(r.GetNodeInfo().GetSize()); - // TFileStat{}; - // const auto id = (FHANDLE)(fh); - // TFileStat stat{id}; if (fh) { nodeid = TFileStat{fh}.INode; } else { - nodeid = TFileStat{fname}.INode; + nodeid = TFileStat{fullName}.INode; } - // GetStatByHandle(stat, id); } break; case NProto::E_DIRECTORY_NODE: { isDir = true; - nodeid = Mkdir(fname); - // id = stat; - // DUMP("mkdir=", mkdires, id); - - // DUMP("dir"); + nodeid = MakeDirectoryRecursive(fullName); } break; case NProto::E_LINK_NODE: - // NFs::HardLink(const TString &existingPath, const TString - // &newPath) NFs::SymLink(const TString &targetPath, const - // TString &linkPath) - // DUMP("link"); + // TODO: NFs::HardLink(const TString &existingPath, const + // TString &newPath) NFs::SymLink(const TString &targetPath, + // const TString &linkPath) break; case NProto::E_SOCK_NODE: - // DUMP("sock"); - break; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + case NProto::E_INVALID_NODE: - break; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); } if (!nodeid) { - nodeid = TFileStat{fname}.INode; + nodeid = TFileStat{fullName}.INode; } - // DUMP(stat.INode, stat.IsDir(), stat.IsFile()); - - // DUMP(id, fh.GetLength(), fh.GetPosition()); - // CreateIfMissing(PathByNode()) if (nodeid) { NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = nodeid; NodePath[nodeid] = PathByNode(parentNode) + r.GetNodeInfo().GetNewNodeName() + (isDir ? "/" : ""); - // DUMP("savepath", nodeid, NodePath[nodeid]); - - // DUMP(NodePath[id], parentNode); } - // DUMP(NodePath); - return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_HANDLE, started, {})); + TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); } - /* - - - / 1 - /dir 2 p=1 - /dir/file 3 p=2 - - - - - - - - */ auto request = CreateRequest(); - // request->SetNodeId(r.GetNodeInfo().GetNodeId()); - // request->SetName(r.GetNodeInfo().GetNodeName()); - // if (r.GetNodeInfo().GetNewParentNodeId() == RootNodeId) { - // request->SetNodeId(r.GetNodeInfo().GetNewParentNodeId()); - //} else - const auto parentNode = NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); if (!parentNode) { - // DUMP("nop", r.GetNodeInfo().GetNewParentNodeId()); return MakeFuture(TCompletedRequest{}); } request->SetNodeId(parentNode); - // DUMP("pppath", PathByNode(parentNode)); - /* - if - (NodesLogToActual.contains(r.GetNodeInfo().GetNewParentNodeId())) { - request->SetNodeId( - NodesLogToActual[r.GetNodeInfo().GetNewParentNodeId()]); - } else { - DUMP( - "unknown parent node", - r.GetNodeInfo().GetNewParentNodeId(), - NodesLogToActual); - return MakeFuture(TCompletedRequest{}); - } - */ auto name = r.GetNodeInfo().GetNewNodeName(); request->SetName(r.GetNodeInfo().GetNewNodeName()); // request->SetGid(); // request->SetUid(); - // DUMP(r.GetNodeInfo().GetType()); switch (r.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: - // DUMP("file"); request->MutableFile()->SetMode(r.GetNodeInfo().GetMode()); break; case NProto::E_DIRECTORY_NODE: - // DUMP("dir"); request->MutableDirectory()->SetMode(r.GetNodeInfo().GetMode()); break; case NProto::E_LINK_NODE: - // DUMP("link"); + // TODO: // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); - return MakeFuture(TCompletedRequest{}); // TODO + return MakeFuture(TCompletedRequest{}); + break; + case NProto::E_SYMLINK_NODE: + // TODO: + // request->MutableSymlink()->SetTargetPath(); break; - - // case NProto::E_ symlink?: - // request->MutableSymlink()->SetTargetPath(); - case NProto::E_SOCK_NODE: - // DUMP("sock"); request->MutableSocket()->SetMode(r.GetNodeInfo().GetMode()); break; case NProto::E_INVALID_NODE: - // DUMP("invv?????"); return MakeFuture( - TCompletedRequest{}); // Do not create files with invalid - // type - too hard to delete them + TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); + // Do not create files with invalid + // type - too hard to delete them break; } - // request->Se - - // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, - // new_node_name=home, mode=509, node_id=12526, size=0} - - // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} - - // DUMP(request); - Cerr << "crreq=" << *request.get() << "\n"; - // auto started = TInstant::Now(); auto self = weak_from_this(); return Session->CreateNode(CreateCallContext(), std::move(request)) .Apply( @@ -1812,39 +974,15 @@ class TReplayRequestGenerator final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { TGuard guard(StateLock); - // Y_ABORT_UNLESS(StagedNodes.erase(name)); try { auto response = future.GetValue(); - - // DUMP("crd",name,r.GetNodeInfo().GetNodeId(),response.GetNode().GetId()); - CheckResponse(response); - - Nodes[name] = TNode{name, response.GetNode()}; - // Nodes[name].Attrs.get - // NodesLogToActual[r.GetNodeInfo().GetNodeId()] = Nodes[name]; + // Nodes[name] = TNode{name, response.GetNode()}; if (response.GetNode().GetId()) { NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = response.GetNode().GetId(); - - /* - DUMP( - "savepath4", - r.GetNodeInfo().GetNodeId(), - NodesLogToLocal[r.GetNodeInfo().GetNodeId()]); - */ - /* - const auto parentNode = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - NodePath[response.GetNode().GetId()] = - NodePath[parentNode] + "/" + - r.GetNodeInfo().GetNewNodeName(); - DUMP(NodePath[response.GetNode().GetId()], - parentNode); - */ } - // DUMP(r.GetNodeInfo().GetNodeId(),response.GetNode().GetId()//,NodesLogToActual); return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; } catch (const TServiceError& e) { @@ -1859,11 +997,19 @@ class TReplayRequestGenerator final } TFuture DoRenameNode( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} + // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, + // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} + // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + auto started = TInstant::Now(); if (Spec.GetNoRead()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); @@ -1872,58 +1018,34 @@ class TReplayRequestGenerator final const auto parentnodeid = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - auto fname = Spec.GetReplayRoot() + - //"/" + - PathByNode(parentnodeid) + - r.GetNodeInfo().GetNodeName(); + auto fullName = Spec.GetReplayRoot() + PathByNode(parentnodeid) + + r.GetNodeInfo().GetNodeName(); const auto newparentnodeid = NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - auto newfname = Spec.GetReplayRoot() + //"/" + - PathByNode(newparentnodeid) + - r.GetNodeInfo().GetNewNodeName(); - // NodePath[0]; - // DUMP("brename", TFileStat{fname}.Size, TFileStat{newfname}.Size); - const auto renameres = NFs::Rename(fname, newfname); - // DUMP("arename", TFileStat{fname}.Size, TFileStat{newfname}.Size); - // DUMP(renameres, fname, newfname); + auto newFullName = Spec.GetReplayRoot() + + PathByNode(newparentnodeid) + + r.GetNodeInfo().GetNewNodeName(); + const auto renameres = NFs::Rename(fullName, newFullName); STORAGE_DEBUG( - "rename " << fname << " => " << newfname << " : " << renameres); - return MakeFuture(TCompletedRequest{}); + "rename " << fullName << " => " << newFullName << " : " + << renameres); + return MakeFuture( + TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); } - /* - if (Nodes.empty()) { - // return DoCreateNode({}); - } - - auto started = TInstant::Now(); - - auto it = Nodes.begin(); - auto old = it->first; - auto newbie = GenerateNodeName(); - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(old); - request->SetNewParentId(RootNodeId); - request->SetNewName(newbie); - - StagedNodes[newbie] = {}; - StagedNodes[old] = std::move(it->second); - Nodes.erase(it); - */ - - auto started = TInstant::Now(); auto request = CreateRequest(); - // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} - // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, - // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} - // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); if (!node) { - return MakeFuture(TCompletedRequest{}); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError( + E_FAIL, + TStringBuilder{} << "Log node " + << r.GetNodeInfo().GetParentNodeId() + << "not found in mappiong")}); } request->SetNodeId(node); request->SetName(r.GetNodeInfo().GetNodeName()); @@ -1932,23 +1054,15 @@ class TReplayRequestGenerator final request->SetNewName(r.GetNodeInfo().GetNewNodeName()); request->SetFlags(r.GetNodeInfo().GetFlags()); - // Cerr << "rename=" << *request << "\n"; auto self = weak_from_this(); return Session->RenameNode(CreateCallContext(), std::move(request)) .Apply( - [= // , - - // old = std::move(old), newbie = std::move(newbie) + [= ](const TFuture& future) { if (auto ptr = self.lock()) { - return ptr->HandleRenameNode( - future, - // old, newbie, - - started, - r); + return ptr->HandleRenameNode(future, started, r); } return TCompletedRequest{ @@ -1960,33 +1074,20 @@ class TReplayRequestGenerator final TCompletedRequest HandleRenameNode( const TFuture& future, - // const TString& old, - // const TString& newbie, TInstant started, const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { TGuard guard(StateLock); - /* - auto node = std::move(StagedNodes[old]); - StagedNodes.erase(old); - StagedNodes.erase(newbie); - */ - // Nodes.erase(r.GetNodeInfo().GetNodeName()); try { auto response = future.GetValue(); CheckResponse(response); - Nodes[r.GetNodeInfo().GetNewNodeName()] = - std::move(Nodes[r.GetNodeInfo().GetNodeName()]); - Nodes.erase(r.GetNodeInfo().GetNodeName()); - // Nodes[newbie] = std::move(node); return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "rename node %s has failed: %s", - r.GetNodeInfo().GetNodeName().c_str(), // old.c_str(), + r.GetNodeInfo().GetNodeName().c_str(), FormatError(error).c_str()); - return {NProto::ACTION_RENAME_NODE, started, error}; } } @@ -1994,18 +1095,20 @@ class TReplayRequestGenerator final TFuture DoUnlinkNode( NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { - if (Spec.GetNoWrite()) { - return {}; - } - // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} - TGuard guard(StateLock); auto started = TInstant::Now(); + if (Spec.GetNoWrite()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); if (UseFs()) { - // DUMP(NodePath); const auto parentNodeId = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); if (!parentNodeId) { @@ -2017,16 +1120,11 @@ class TReplayRequestGenerator final started, MakeError(E_CANCELLED, "cancelled"))); } - const auto fname = Spec.GetReplayRoot() + "/" + - PathByNode(parentNodeId) + - r.GetNodeInfo().GetNodeName(); - // DUMP(r.GetNodeInfo().GetParentNodeId(),PathByNode(NodeIdMapped(r.GetNodeInfo().GetParentNodeId()))); - - // {parent_node_id=3, node_name=tfrgYZ1} - // const auto unlinkres = unlink(fname.c_str()); - const auto unlinkres = NFs::Remove(fname); - STORAGE_DEBUG("unlink " << fname << " = " << unlinkres); - // DUMP(unlinkres, fname); + const auto fullName = Spec.GetReplayRoot() + "/" + + PathByNode(parentNodeId) + + r.GetNodeInfo().GetNodeName(); + const auto unlinkres = NFs::Remove(fullName); + STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); // TODO : // NodesLogToActual.erase(...) // NodePath.erase(...) @@ -2034,33 +1132,14 @@ class TReplayRequestGenerator final TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); } - /* - if (Nodes.empty()) { - // return DoCreateNode(); - } - */ - - /* - auto it = Nodes.begin(); - auto name = it->first; - */ - auto name = r.GetNodeInfo().GetNodeName(); auto request = CreateRequest(); - // request->SetNodeId(RootNodeId); request->SetName(name); - // request.set - // request->SetNodeId(r.GetNodeInfo().GetNodeId()); - // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{}); } request->SetNodeId(node); - /* - StagedNodes[name] = std::move(it->second); - Nodes.erase(it); - */ auto self = weak_from_this(); return Session->UnlinkNode(CreateCallContext(), std::move(request)) .Apply( @@ -2083,10 +1162,6 @@ class TReplayRequestGenerator final const TString& name, TInstant started) { - with_lock (StateLock) { - StagedNodes.erase(name); - } - try { auto response = future.GetValue(); CheckResponse(response); @@ -2102,77 +1177,10 @@ class TReplayRequestGenerator final return {NProto::ACTION_REMOVE_NODE, started, error}; } } - /* - TFuture DoCreateHandle2() - { - static const int flags = - ProtoFlag(NProto::TCreateHandleRequest::E_READ) | - ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); - - TGuard guard(StateLock); - if (Nodes.empty()) { - // return DoCreateNode(); - } - - auto started = TInstant::Now(); - - auto it = Nodes.begin(); - if (Nodes.size() > 1) { - std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); - } - - auto name = it->first; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - request->SetFlags(flags); - - auto self = weak_from_this(); - return Session->CreateHandle(CreateCallContext(), - std::move(request)) .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateHandle2(future, name, - started); - } - - return TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleCreateHandle2( - const TFuture& future, - const TString name, - TInstant started) - { - TGuard guard(StateLock); - - try { - auto response = future.GetValue(); - CheckResponse(response); - - auto handle = response.GetHandle(); - Handles[handle] = THandle{name, handle}; - - return {NProto::ACTION_CREATE_HANDLE, started, - response.GetError()}; } catch (const TServiceError& e) { auto error = - MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "create - handle for %s has failed: %s", name.c_str(), FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_HANDLE, started, error}; - } - } - */ TFuture DoDestroyHandle( NCloud::NFileStore::NProto::TProfileLogRequestInfo r) { - // DestroyHandle 0.002475s S_OK {node_id=10, + // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} TGuard guard(StateLock); auto started = TInstant::Now(); @@ -2196,14 +1204,11 @@ class TReplayRequestGenerator final auto& fhandle = it->second; const auto len = fhandle.GetLength(); const auto pos = fhandle.GetPosition(); - // auto closed = fhandle.Close(); OpenHandles.erase(handleid); HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); STORAGE_DEBUG( - "Close " << handleid << " orig=" - << r.GetNodeInfo().GetHandle() - //<< " closed=" << closed + "Close " << handleid << " orig=" << r.GetNodeInfo().GetHandle() << " pos=" << pos << " len=" << len << " open map size=" << OpenHandles.size() << " map size=" << HandlesLogToActual.size()); @@ -2211,27 +1216,6 @@ class TReplayRequestGenerator final TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, started, {})); } - /* - if (Handles.empty()) { - return DoCreateHandle(); - } - - auto started = TInstant::Now(); - auto it = Handles.begin(); - - ui64 handle = it->first; - auto name = it->second.Path; - Handles.erase(it); - if (auto it = Locks.find(handle); it != Locks.end()) { - Locks.erase(it); - } - if (auto it = StagedLocks.find(handle); it != StagedLocks.end()) - { StagedLocks.erase(it); - } - - auto request = CreateRequest(); - request->SetHandle(handle); - */ auto name = r.GetNodeInfo().GetNodeName(); auto request = CreateRequest(); @@ -2265,17 +1249,20 @@ class TReplayRequestGenerator final TFuture DoGetNodeAttr( NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { + const auto started = TInstant::Now(); if (Spec.GetNoRead()) { - return {}; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto started = TInstant::Now(); // TODO: by parent + name // // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} - // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} @@ -2283,12 +1270,6 @@ class TReplayRequestGenerator final if (r.GetNodeInfo().GetNodeName()) { KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name = r.GetNodeInfo().GetNodeName(); - /* - DUMP( - "saveknown1", - r.GetNodeInfo().GetNodeId(), - r.GetNodeInfo().GetNodeName()); - */ } if (r.GetNodeInfo().GetParentNodeId() && r.GetNodeInfo().GetParentNodeId() != @@ -2296,18 +1277,10 @@ class TReplayRequestGenerator final { KnownLogNodes[r.GetNodeInfo().GetNodeId()].ParentLog = r.GetNodeInfo().GetParentNodeId(); - /* - DUMP( - "saveknown2", - r.GetNodeInfo().GetNodeId(), - r.GetNodeInfo().GetParentNodeId()); - */ } // TODO: can create and truncate to size here missing files - // DUMP(KnownNodes); - const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); if (!nodeid) { @@ -2319,39 +1292,10 @@ class TReplayRequestGenerator final auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); [[maybe_unused]] const auto stat = TFileStat{fname}; - // DUMP("gna", fname, stat.INode); - return MakeFuture( TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); } - /* - if (Nodes.empty()) { - //return DoCreateNode(); - } - */ - /* - auto it = Nodes.begin(); - if (Nodes.size() > 1) { - std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); - } - - auto name = it->first; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - */ - - /* - if - (!NodesLogToActual.contains(r.GetNodeInfo().GetParentNodeId())) { - DUMP("skip nodeattr"); - return MakeFuture(TCompletedRequest{}); - } - */ auto request = CreateRequest(); - // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} - // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{}); @@ -2359,18 +1303,7 @@ class TReplayRequestGenerator final request->SetNodeId(node); auto name = r.GetNodeInfo().GetNodeName(); request->SetName(r.GetNodeInfo().GetNodeName()); - // request->SetFlags(r.GetNodeInfo().GetFlags()); request->SetFlags(r.GetNodeInfo().GetFlags()); - /* - DUMP( - "ids?", - r.GetNodeInfo().GetParentNodeId(), - NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]); - if (!NodesLogToActual[r.GetNodeInfo().GetParentNodeId()]) { - DUMP(NodesLogToActual); - } - */ - Cerr << "attr=" << *request << "\n"; auto self = weak_from_this(); STORAGE_DEBUG("GetNodeAttr client started"); return Session->GetNodeAttr(CreateCallContext(), std::move(request)) @@ -2399,21 +1332,6 @@ class TReplayRequestGenerator final STORAGE_DEBUG("GetNodeAttr client completed"); CheckResponse(response); TGuard guard(StateLock); - /* - if (response.GetNode().SerializeAsString() != - Nodes[name].Attrs.SerializeAsString()) - { - auto error = MakeError( - E_FAIL, - TStringBuilder() - << "attributes are not equal for node " << - name << ": " - << response.GetNode().DebugString().Quote() - << " != " << - Nodes[name].Attrs.DebugString().Quote()); - STORAGE_ERROR(error.GetMessage().c_str()); - } - */ return {NProto::ACTION_GET_NODE_ATTR, started, {}}; } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); @@ -2423,7 +1341,6 @@ class TReplayRequestGenerator final FormatError(error).c_str()); return {NProto::ACTION_GET_NODE_ATTR, started, {}}; - // return {NProto::ACTION_GET_NODE_ATTR, started, error}; } } @@ -2464,7 +1381,10 @@ class TReplayRequestGenerator final } if (it == Handles.end()) { - // return DoCreateHandle(); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}); } auto handle = it->first; @@ -2595,39 +1515,6 @@ class TReplayRequestGenerator final return request; } - TString GenerateNodeName() - { - return TStringBuilder() - << Headers.GetClientId() << ":" << CreateGuidAsString(); - } - - THandleInfo GetHandleInfo() - { - Y_ABORT_UNLESS(!HandleInfos.empty()); - ui64 index = RandomNumber(HandleInfos.size()); - std::swap(HandleInfos[index], HandleInfos.back()); - - auto handle = HandleInfos.back(); - HandleInfos.pop_back(); - - return handle; - } - - THandleInfo GetHandleInfo(ui64 id) - { - return HandleInfos[id]; - /* - Y_ABORT_UNLESS(!HandleInfos.empty()); - ui64 index = RandomNumber(HandleInfos.size()); - std::swap(HandleInfos[index], HandleInfos.back()); - - auto handle = HandleInfos.back(); - HandleInfos.pop_back(); - - return handle; - */ - } - template void CheckResponse(const T& response) { @@ -2641,19 +1528,6 @@ class TReplayRequestGenerator final return MakeIntrusive( LastRequestId.fetch_add(1, std::memory_order_relaxed)); } - /* - ui64 PickSlot(THandleInfo& handleInfo, ui64 reqBytes) - { - const ui64 slotCount = handleInfo.Size / reqBytes; - Y_ABORT_UNLESS(slotCount); - if (Spec.GetSequential()) { - return handleInfo.LastSlot = (handleInfo.LastSlot + 1) % - slotCount; - } - - return RandomNumber(slotCount); - } - */ }; } // namespace diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp index 09fd751480..d9d4915a84 100644 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -326,8 +326,8 @@ class TLoadTest final "%s request sent: %s %lu %lu", MakeTestTag().c_str(), SessionId.c_str(), - SeqNo, RequestsSent); - //DUMP("re"); + SeqNo, + RequestsSent); } DUMP("fin"); //PrintBackTrace(); @@ -582,23 +582,20 @@ class TLoadTest final return false; } - ++CurrentIoDepth; auto self = weak_from_this(); - const auto future = RequestGenerator->ExecuteNextRequest(); - // DUMP(future.Initialized(), CurrentIoDepth); - if (!future.Initialized()) { - return true; - } - future.Apply( + ++CurrentIoDepth; + RequestGenerator->ExecuteNextRequest().Apply( [=](const TFuture& future) { - if (!future.Initialized()) { - return; - } if (auto ptr = self.lock()) { ptr->SignalCompletion(future.GetValue()); } }); + + if (RequestGenerator->InstantProcessQueue()) { + ProcessCompletedRequests(); + } + return true; } From 83339bd79441a476d3947cbda3bd0bbd272351e2 Mon Sep 17 00:00:00 2001 From: proller Date: Mon, 23 Sep 2024 19:30:16 +0000 Subject: [PATCH 04/44] clean --- .../tools/testing/replay/lib/request_data.cpp | 639 ---------------- .../testing/replay/lib/request_index.cpp | 684 ------------------ .../tools/testing/replay/lib/ya.make | 2 - 3 files changed, 1325 deletions(-) delete mode 100644 cloud/filestore/tools/testing/replay/lib/request_data.cpp delete mode 100644 cloud/filestore/tools/testing/replay/lib/request_index.cpp diff --git a/cloud/filestore/tools/testing/replay/lib/request_data.cpp b/cloud/filestore/tools/testing/replay/lib/request_data.cpp deleted file mode 100644 index 2a8247b102..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/request_data.cpp +++ /dev/null @@ -1,639 +0,0 @@ -#include "request.h" - -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -using namespace NThreading; -using namespace NCloud::NFileStore::NClient; - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -const ui64 SEGMENT_SIZE = 128; - -struct TSegment -{ - ui64 Handle = 0; - ui64 LastWriteRequestId = 0; -}; - -static_assert(sizeof(TSegment) <= SEGMENT_SIZE); - -bool Compare( - const TSegment& expected, - const TSegment& actual, - TString* message) -{ - TStringBuilder sb; - - if (expected.Handle != actual.Handle) { - sb << "expected.Handle != actual.Handle: " - << expected.Handle << " != " << actual.Handle; - } - - if (expected.LastWriteRequestId != actual.LastWriteRequestId) { - if (sb.Size()) { - sb << ", "; - } - - sb << "expected.LastWriteRequestId != actual.LastWriteRequestId: " - << expected.LastWriteRequestId - << " != " << actual.LastWriteRequestId; - } - - *message = sb; - - return sb.Empty(); -} - -struct TSegments: TVector, TAtomicRefCount -{ -}; - -using TSegmentsPtr = TIntrusivePtr; - -//////////////////////////////////////////////////////////////////////////////// - -struct THandleInfo -{ - TString Name; - ui64 Handle = 0; - ui64 Size = 0; - TSegmentsPtr Segments; - ui64 LastSlot = 0; - - THandleInfo() = default; - - THandleInfo( - TString name, - ui64 handle, - ui64 size, - TSegmentsPtr segments) noexcept - : Name(std::move(name)) - , Handle(handle) - , Size(size) - , Segments(std::move(segments)) - {} -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TDataRequestGenerator final - : public IRequestGenerator - , public std::enable_shared_from_this -{ -private: - static constexpr ui32 DefaultBlockSize = 4_KB; - - const NProto::TDataLoadSpec Spec; - const TString FileSystemId; - const NProto::THeaders Headers; - - TLog Log; - - ISessionPtr Session; - - TVector> Actions; - ui64 TotalRate = 0; - - TMutex StateLock; - ui64 LastWriteRequestId = 0; - - TVector IncompleteHandleInfos; - TVector HandleInfos; - - ui64 ReadBytes = DefaultBlockSize; - ui64 WriteBytes = DefaultBlockSize; - double AppendProbability = 1; - ui64 BlockSize = DefaultBlockSize; - ui64 InitialFileSize = 0; - - std::atomic LastRequestId = 0; - -public: - TDataRequestGenerator( - NProto::TDataLoadSpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) - : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) - , Headers(std::move(headers)) - , Session(std::move(session)) - { - Log = logging->CreateLog(Headers.GetClientId()); - - if (auto size = Spec.GetBlockSize()) { - BlockSize = size; - Y_ENSURE(BlockSize % DefaultBlockSize == 0); - } - if (auto bytes = Spec.GetReadBytes()) { - ReadBytes = bytes; - } - if (auto bytes = Spec.GetWriteBytes()) { - WriteBytes = bytes; - } - if (auto p = Spec.GetAppendPercentage()) { - AppendProbability = p / 100.; - } - - InitialFileSize = Spec.GetInitialFileSize(); - - if (Spec.GetValidationEnabled()) { - Y_ENSURE( - InitialFileSize % SEGMENT_SIZE == 0, - Sprintf( - "InitialFileSize (%lu) %% SEGMENT_SIZE (%lu) != 0", - InitialFileSize, - SEGMENT_SIZE - ) - ); - - Y_ENSURE( - WriteBytes % SEGMENT_SIZE == 0, - Sprintf( - "WriteBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", - WriteBytes, - SEGMENT_SIZE - ) - ); - - Y_ENSURE( - ReadBytes % SEGMENT_SIZE == 0, - Sprintf( - "ReadBytes (%lu) %% SEGMENT_SIZE (%lu) != 0", - ReadBytes, - SEGMENT_SIZE - ) - ); - } - - for (const auto& action: Spec.GetActions()) { - Y_ENSURE(action.GetRate() > 0, "please specify positive action rate"); - - TotalRate += action.GetRate(); - Actions.emplace_back(std::make_pair(TotalRate, action.GetAction())); - } - - Y_ENSURE(!Actions.empty(), "please specify at least one action for the test spec"); - } - - bool HasNextRequest() override - { - return true; - } - - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - - NThreading::TFuture ExecuteNextRequest() override - { - const auto& action = PeekNextAction(); - switch (action) { - case NProto::ACTION_READ: - return DoRead(); - case NProto::ACTION_WRITE: - return DoWrite(); - default: - Y_ABORT("unexpected action: %u", (ui32)action); - } - } - -private: - NProto::EAction PeekNextAction() - { - auto number = RandomNumber(TotalRate); - auto it = LowerBound( - Actions.begin(), - Actions.end(), - number, - [] (const auto& pair, ui64 b) { return pair.first < b; }); - - Y_ABORT_UNLESS(it != Actions.end()); - return it->second; - } - - TFuture DoCreateHandle() - { - static const int flags = ProtoFlag(NProto::TCreateHandleRequest::E_CREATE) - | ProtoFlag(NProto::TCreateHandleRequest::E_EXCLUSIVE) - | ProtoFlag(NProto::TCreateHandleRequest::E_READ) - | ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); - - auto started = TInstant::Now(); - TGuard guard(StateLock); - auto name = GenerateNodeName(); - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - request->SetFlags(flags); - - auto self = weak_from_this(); - return Session->CreateHandle(CreateCallContext(), std::move(request)).Apply( - [=, name = std::move(name)] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleCreateHandle(future, name, started); - } - - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "cancelled")}); - }); - } - - TFuture HandleCreateHandle( - const TFuture& future, - const TString& name, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - auto handle = response.GetHandle(); - auto& infos = InitialFileSize ? IncompleteHandleInfos : HandleInfos; - with_lock (StateLock) { - TSegmentsPtr segments; - - if (Spec.GetValidationEnabled()) { - segments = new TSegments(); - segments->resize(Spec.GetInitialFileSize() / SEGMENT_SIZE); - } - - infos.emplace_back(name, handle, 0, std::move(segments)); - } - - NThreading::TFuture setAttr; - if (InitialFileSize) { - static const int flags = - ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetNodeId(response.GetNodeAttr().GetId()); - request->SetFlags(flags); - request->MutableUpdate()->SetSize(InitialFileSize); - - setAttr = Session->SetNodeAttr(CreateCallContext(), std::move(request)); - } else { - setAttr = NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); - } - - return setAttr.Apply( - [=] (const TFuture& f) { - return HandleResizeAfterCreateHandle(f, name, started); - }); - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("create handle for %s has failed: %s", - name.Quote().c_str(), - FormatError(error).c_str()); - - return NThreading::MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - error - }); - } - } - - TCompletedRequest HandleResizeAfterCreateHandle( - const TFuture& future, - const TString& name, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - bool handleFound = false; - with_lock (StateLock) { - ui32 handleIdx = 0; - while (handleIdx < IncompleteHandleInfos.size()) { - auto& hinfo = IncompleteHandleInfos[handleIdx]; - if (hinfo.Name == name) { - handleFound = true; - - hinfo.Size = Max( - hinfo.Size, - response.GetNode().GetSize()); - - STORAGE_INFO( - "updated file size for handle %lu and file %s" - ", size=%lu", - hinfo.Handle, - name.Quote().c_str(), - hinfo.Size); - - break; - } - - ++handleIdx; - } - - if (handleIdx < IncompleteHandleInfos.size()) { - DoSwap( - IncompleteHandleInfos[handleIdx], - IncompleteHandleInfos.back()); - - HandleInfos.push_back( - std::move(IncompleteHandleInfos.back())); - - IncompleteHandleInfos.pop_back(); - } - } - - if (!handleFound) { - STORAGE_WARN("handle for file %s not found", - name.Quote().c_str()); - } - - return { - NProto::ACTION_CREATE_HANDLE, - started, - response.GetError() - }; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("set node attr handle for %s has failed: %s", - name.Quote().c_str(), - FormatError(error).c_str()); - - return { - NProto::ACTION_CREATE_HANDLE, - started, - error - }; - } - } - - TFuture DoRead() - { - TGuard guard(StateLock); - if (HandleInfos.empty()) { - return DoCreateHandle(); - } - - auto handleInfo = GetHandleInfo(); - if (handleInfo.Size < ReadBytes) { - return DoWrite(handleInfo); - } - - const auto started = TInstant::Now(); - const ui64 slotOffset = PickSlot(handleInfo, ReadBytes); - const ui64 byteOffset = slotOffset * ReadBytes; - - auto request = CreateRequest(); - request->SetHandle(handleInfo.Handle); - request->SetOffset(byteOffset); - request->SetLength(ReadBytes); - - auto self = weak_from_this(); - return Session->ReadData(CreateCallContext(), std::move(request)).Apply( - [=] (const TFuture& future){ - if (auto ptr = self.lock()) { - return ptr->HandleRead( - future, - handleInfo, - started, - byteOffset - ); - } - - return TCompletedRequest{ - NProto::ACTION_READ, - started, - MakeError(E_FAIL, "cancelled")}; - }); - } - - TCompletedRequest HandleRead( - const TFuture& future, - THandleInfo handleInfo, - TInstant started, - ui64 byteOffset) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - const auto& buffer = response.GetBuffer(); - - ui64 segmentId = byteOffset / SEGMENT_SIZE; - for (ui64 offset = 0; offset < ReadBytes; offset += SEGMENT_SIZE) { - const TSegment* segment = - reinterpret_cast(buffer.data() + offset); - if (Spec.GetValidationEnabled()) { - Y_ABORT_UNLESS(handleInfo.Segments); - - auto& segments = *handleInfo.Segments; - Y_ABORT_UNLESS(segmentId < segments.size()); - - TString message; - if (!Compare(segments[segmentId], *segment, &message)) { - throw TServiceError(E_FAIL) - << Sprintf("Validation failed: %s", message.c_str()); - } - } - - ++segmentId; - } - - with_lock (StateLock) { - HandleInfos.emplace_back(std::move(handleInfo)); - } - - return {NProto::ACTION_READ, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("read for %s has failed: %s", - handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_READ, started, error}; - } - } - - TFuture DoWrite(THandleInfo handleInfo = {}) - { - TGuard guard(StateLock); - if (!handleInfo.Handle) { - if (HandleInfos.empty()) { - return DoCreateHandle(); - } - - handleInfo = GetHandleInfo(); - } - - const auto started = TInstant::Now(); - ui64 byteOffset = handleInfo.Size; - if (RandomNumber() < AppendProbability) { - handleInfo.Size += WriteBytes; - } else { - handleInfo.Size = Max(handleInfo.Size, WriteBytes); - const ui64 slotOffset = PickSlot(handleInfo, WriteBytes); - byteOffset = slotOffset * WriteBytes; - } - - auto request = CreateRequest(); - request->SetHandle(handleInfo.Handle); - request->SetOffset(byteOffset); - - ++LastWriteRequestId; - - TString buffer(WriteBytes, '\0'); - ui64 segmentId = byteOffset / SEGMENT_SIZE; - for (ui64 offset = 0; offset < WriteBytes; offset += SEGMENT_SIZE) { - TSegment* segment = - reinterpret_cast(buffer.begin() + offset); - segment->Handle = handleInfo.Handle; - segment->LastWriteRequestId = LastWriteRequestId; - - if (Spec.GetValidationEnabled()) { - Y_ABORT_UNLESS(handleInfo.Segments); - - auto& segments = *handleInfo.Segments; - if (segments.size() <= segmentId) { - segments.emplace_back(); - } - - segments[segmentId] = *segment; - } - - ++segmentId; - } - *request->MutableBuffer() = std::move(buffer); - - auto self = weak_from_this(); - return Session->WriteData(CreateCallContext(), std::move(request)).Apply( - [=] (const TFuture& future){ - if (auto ptr = self.lock()) { - return ptr->HandleWrite(future, handleInfo, started); - } - - return TCompletedRequest{ - NProto::ACTION_WRITE, - started, - MakeError(E_FAIL, "cancelled")}; - }); - - } - - TCompletedRequest HandleWrite( - const TFuture& future, - THandleInfo handleInfo, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - with_lock (StateLock) { - HandleInfos.emplace_back(std::move(handleInfo)); - } - - return {NProto::ACTION_WRITE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("write on %s has failed: %s", - handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_WRITE, started, error}; - } - } - - template - std::shared_ptr CreateRequest() - { - auto request = std::make_shared(); - request->SetFileSystemId(FileSystemId); - request->MutableHeaders()->CopyFrom(Headers); - - return request; - } - - TString GenerateNodeName() - { - return TStringBuilder() << Headers.GetClientId() << ":" << CreateGuidAsString(); - } - - THandleInfo GetHandleInfo() - { - Y_ABORT_UNLESS(!HandleInfos.empty()); - ui64 index = RandomNumber(HandleInfos.size()); - std::swap(HandleInfos[index], HandleInfos.back()); - - auto handle = HandleInfos.back(); - HandleInfos.pop_back(); - - return handle; - } - - template - void CheckResponse(const T& response) - { - if (HasError(response)) { - throw TServiceError(response.GetError()); - } - } - - TIntrusivePtr CreateCallContext() - { - return MakeIntrusive( - LastRequestId.fetch_add(1, std::memory_order_relaxed)); - } - - ui64 PickSlot(THandleInfo& handleInfo, ui64 reqBytes) - { - const ui64 slotCount = handleInfo.Size / reqBytes; - Y_ABORT_UNLESS(slotCount); - if (Spec.GetSequential()) { - return handleInfo.LastSlot = (handleInfo.LastSlot + 1) % slotCount; - } - - return RandomNumber(slotCount); - } -}; - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -IRequestGeneratorPtr CreateDataRequestGenerator( - NProto::TDataLoadSpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) -{ - return std::make_shared( - std::move(spec), - std::move(logging), - std::move(session), - std::move(filesystemId), - std::move(headers)); -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request_index.cpp b/cloud/filestore/tools/testing/replay/lib/request_index.cpp deleted file mode 100644 index f6bf0cccb5..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/request_index.cpp +++ /dev/null @@ -1,684 +0,0 @@ -#include "request.h" - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -namespace NCloud::NFileStore::NReplay { - -using namespace NThreading; -using namespace NCloud::NFileStore::NClient; - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -class TIndexRequestGenerator final - : public IRequestGenerator - , public std::enable_shared_from_this -{ -private: - static constexpr ui32 LockLength = 4096; - - struct TNode - { - TString Name; - NProto::TNodeAttr Attrs; - }; - - struct THandle - { - TString Path; - ui64 Handle = 0; - }; - -private: - const NProto::TIndexLoadSpec Spec; - const TString FileSystemId; - const NProto::THeaders Headers; - const ui64 OwnerId = RandomNumber(100500u); - - TLog Log; - - ISessionPtr Session; - - TVector> Actions; - ui64 TotalRate = 0; - - TMutex StateLock; - THashMap StagedNodes; - THashMap Nodes; - - THashMap Handles; - - THashSet Locks; - THashSet StagedLocks; - - ui64 LastRequestId = 0; - -public: - TIndexRequestGenerator( - NProto::TIndexLoadSpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) - : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) - , Headers(std::move(headers)) - , Session(std::move(session)) - { - Log = logging->CreateLog(Headers.GetClientId()); - - for (const auto& action: Spec.GetActions()) { - Y_ENSURE(action.GetRate() > 0, "please specify positive action rate"); - - TotalRate += action.GetRate(); - Actions.emplace_back(std::make_pair(TotalRate, action.GetAction())); - } - - Y_ENSURE(!Actions.empty(), "please specify at least one action for the test spec"); - } - - bool HasNextRequest() override - { - return true; - } - - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - - NThreading::TFuture ExecuteNextRequest() override - { - const auto& action = PeekNextAction(); - switch (action) { - case NProto::ACTION_CREATE_NODE: - return DoCreateNode(); - case NProto::ACTION_RENAME_NODE: - return DoRenameNode(); - case NProto::ACTION_REMOVE_NODE: - return DoUnlinkNode(); - case NProto::ACTION_CREATE_HANDLE: - return DoCreateHandle(); - case NProto::ACTION_DESTROY_HANDLE: - return DoDestroyHandle(); - case NProto::ACTION_GET_NODE_ATTR: - return DoGetNodeAttr(); - case NProto::ACTION_ACQUIRE_LOCK: - return DoAcquireLock(); - case NProto::ACTION_RELEASE_LOCK: - return DoReleaseLock(); - default: - Y_ABORT("unexpected action: %u", (ui32)action); - } - } - -private: - NProto::EAction PeekNextAction() - { - auto number = RandomNumber(TotalRate); - auto it = LowerBound( - Actions.begin(), - Actions.end(), - number, - [] (const auto& pair, ui64 b) { return pair.first < b; }); - - Y_ABORT_UNLESS(it != Actions.end()); - return it->second; - } - - TFuture DoCreateNode() - { - TGuard guard(StateLock); - - auto name = GenerateNodeName(); - StagedNodes[name] = {}; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - request->MutableFile()->SetMode(0777); - - auto started = TInstant::Now(); - auto self = weak_from_this(); - return Session->CreateNode(CreateCallContext(), std::move(request)).Apply( - [=] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleCreateNode(future, name, started); - } - - return TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleCreateNode( - const TFuture& future, - const TString& name, - TInstant started) - { - TGuard guard(StateLock); - Y_ABORT_UNLESS(StagedNodes.erase(name)); - - try { - auto response = future.GetValue(); - CheckResponse(response); - - Nodes[name] = TNode{name, response.GetNode()}; - return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("create node %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_NODE, started, error}; - } - } - - TFuture DoRenameNode() - { - TGuard guard(StateLock); - - if (Nodes.empty()) { - return DoCreateNode(); - } - - auto started = TInstant::Now(); - - auto it = Nodes.begin(); - auto old = it->first; - auto newbie = GenerateNodeName(); - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(old); - request->SetNewParentId(RootNodeId); - request->SetNewName(newbie); - - StagedNodes[newbie] = {}; - StagedNodes[old] = std::move(it->second); - Nodes.erase(it); - - auto self = weak_from_this(); - return Session->RenameNode(CreateCallContext(), std::move(request)).Apply( - [=, old = std::move(old), newbie = std::move(newbie)] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleRenameNode(future, old, newbie, started); - } - - return TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleRenameNode( - const TFuture& future, - const TString& old, - const TString& newbie, - TInstant started) - { - TGuard guard(StateLock); - - auto node = std::move(StagedNodes[old]); - StagedNodes.erase(old); - StagedNodes.erase(newbie); - - try { - auto response = future.GetValue(); - CheckResponse(response); - - Nodes[newbie] = std::move(node); - return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("rename node %s has failed: %s", - old.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_RENAME_NODE, started, error}; - } - } - - TFuture DoUnlinkNode() - { - TGuard guard(StateLock); - if (Nodes.empty()) { - return DoCreateNode(); - } - - auto started = TInstant::Now(); - - auto it = Nodes.begin(); - auto name = it->first; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - - StagedNodes[name] = std::move(it->second); - Nodes.erase(it); - - auto self = weak_from_this(); - return Session->UnlinkNode(CreateCallContext(), std::move(request)).Apply( - [=, name = std::move(name)] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleUnlinkNode(future, name, started); - } - - return TCompletedRequest{ - NProto::ACTION_REMOVE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleUnlinkNode( - const TFuture& future, - const TString& name, - TInstant started) - { - with_lock (StateLock) { - StagedNodes.erase(name); - } - - try { - auto response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("unlink for %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_REMOVE_NODE, started, error}; - } - } - - TFuture DoCreateHandle() - { - static const int flags = ProtoFlag(NProto::TCreateHandleRequest::E_READ) - | ProtoFlag(NProto::TCreateHandleRequest::E_WRITE); - - TGuard guard(StateLock); - if (Nodes.empty()) { - return DoCreateNode(); - } - - auto started = TInstant::Now(); - - auto it = Nodes.begin(); - if (Nodes.size() > 1) { - std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); - } - - auto name = it->first; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - request->SetFlags(flags); - - auto self = weak_from_this(); - return Session->CreateHandle(CreateCallContext(), std::move(request)).Apply( - [=, name = std::move(name)] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleCreateHandle(future, name, started); - } - - return TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleCreateHandle( - const TFuture& future, - const TString name, - TInstant started) - { - TGuard guard(StateLock); - - try { - auto response = future.GetValue(); - CheckResponse(response); - - auto handle = response.GetHandle(); - Handles[handle] = THandle{name, handle}; - - return {NProto::ACTION_CREATE_HANDLE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("create handle for %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_HANDLE, started, error}; - } - } - - TFuture DoDestroyHandle() - { - TGuard guard(StateLock); - if (Handles.empty()) { - return DoCreateHandle(); - } - - auto started = TInstant::Now(); - auto it = Handles.begin(); - - ui64 handle = it->first; - auto name = it->second.Path; - Handles.erase(it); - if (auto it = Locks.find(handle); it != Locks.end()) { - Locks.erase(it); - } - if (auto it = StagedLocks.find(handle); it != StagedLocks.end()) { - StagedLocks.erase(it); - } - - auto request = CreateRequest(); - request->SetHandle(handle); - - auto self = weak_from_this(); - return Session->DestroyHandle(CreateCallContext(), std::move(request)).Apply( - [=, name = std::move(name)] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleDestroyHandle(name, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_DESTROY_HANDLE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TFuture DoGetNodeAttr() - { - TGuard guard(StateLock); - if (Nodes.empty()) { - return DoCreateNode(); - } - - auto started = TInstant::Now(); - auto it = Nodes.begin(); - if (Nodes.size() > 1) { - std::advance(it, Min(RandomNumber(Nodes.size() - 1), 64lu)); - } - - auto name = it->first; - - auto request = CreateRequest(); - request->SetNodeId(RootNodeId); - request->SetName(name); - - auto self = weak_from_this(); - STORAGE_DEBUG("GetNodeAttr client started"); - return Session->GetNodeAttr(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleGetNodeAttr(name, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleGetNodeAttr( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - STORAGE_DEBUG("GetNodeAttr client completed"); - CheckResponse(response); - TGuard guard(StateLock); - if (response.GetNode().SerializeAsString() != - Nodes[name].Attrs.SerializeAsString()) - { - auto error = MakeError( - E_FAIL, - TStringBuilder() - << "attributes are not equal for node " << name << ": " - << response.GetNode().DebugString().Quote() - << " != " << Nodes[name].Attrs.DebugString().Quote()); - STORAGE_ERROR(error.GetMessage().c_str()); - } - return {NProto::ACTION_GET_NODE_ATTR, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "get node attr %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_GET_NODE_ATTR, started, error}; - } - } - - TCompletedRequest HandleDestroyHandle( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_DESTROY_HANDLE, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("destroy handle %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_DESTROY_HANDLE, started, error}; - } - } - - TFuture DoAcquireLock() - { - TGuard guard(StateLock); - if (Handles.empty()) { - return DoCreateHandle(); - } - - auto started = TInstant::Now(); - auto it = Handles.begin(); - while (it != Handles.end() && (Locks.contains(it->first) || StagedLocks.contains(it->first))) { - ++it; - } - - if (it == Handles.end()) { - return DoCreateHandle(); - } - - auto handle = it->first; - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto self = weak_from_this(); - return Session->AcquireLock(CreateCallContext(), std::move(request)).Apply( - [=] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleAcquireLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleAcquireLock( - ui64 handle, - const TFuture& future, - TInstant started) - { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it == StagedLocks.end()) { - // nothing todo, file was removed - Y_ABORT_UNLESS(!Locks.contains(handle)); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } - - StagedLocks.erase(it); - - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - Locks.insert(handle); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("acquire lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_ACQUIRE_LOCK, started, error}; - } - } - - TFuture DoReleaseLock() - { - TGuard guard(StateLock); - if (Locks.empty()) { - return DoAcquireLock(); - } - - auto it = Locks.begin(); - auto handle = *it; - - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - Locks.erase(it); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto started = TInstant::Now(); - auto self = weak_from_this(); - return Session->ReleaseLock(CreateCallContext(), std::move(request)).Apply( - [=] (const TFuture& future) { - if (auto ptr = self.lock()) { - return ptr->HandleReleaseLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_RELEASE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleReleaseLock( - ui64 handle, - const TFuture& future, - TInstant started) - { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it != StagedLocks.end()) { - StagedLocks.erase(it); - } - - try { - CheckResponse(future.GetValue()); - return {NProto::ACTION_RELEASE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR("release lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_RELEASE_LOCK, started, error}; - } - } - - template - std::shared_ptr CreateRequest() - { - auto request = std::make_shared(); - request->SetFileSystemId(FileSystemId); - request->MutableHeaders()->CopyFrom(Headers); - - return request; - } - - TString GenerateNodeName() - { - return TStringBuilder() << Headers.GetClientId() << ":" << CreateGuidAsString(); - } - - template - void CheckResponse(const T& response) - { - if (HasError(response)) { - throw TServiceError(response.GetError()); - } - } - - TIntrusivePtr CreateCallContext() - { - return MakeIntrusive(AtomicIncrement(LastRequestId)); - } -}; - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -IRequestGeneratorPtr CreateIndexRequestGenerator( - NProto::TIndexLoadSpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) -{ - return std::make_shared( - std::move(spec), - std::move(logging), - std::move(session), - std::move(filesystemId), - std::move(headers)); -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/ya.make b/cloud/filestore/tools/testing/replay/lib/ya.make index 5851626fdc..c24c4d3757 100644 --- a/cloud/filestore/tools/testing/replay/lib/ya.make +++ b/cloud/filestore/tools/testing/replay/lib/ya.make @@ -4,8 +4,6 @@ SRCS( client.cpp context.h executor.cpp - #request_data.cpp - #request_index.cpp request_replay.cpp test.cpp ) From b503a1e9377cbca4b5fa9c8cd561a21c62f03d0c Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 24 Sep 2024 01:15:14 +0000 Subject: [PATCH 05/44] clean --- .../tools/testing/replay/lib/request.h | 10 +- ...quest_replay.cpp => request_replay_fs.cpp} | 900 ++++++------------ .../tools/testing/replay/lib/test.cpp | 36 +- .../tools/testing/replay/lib/ya.make | 3 +- .../tools/testing/replay/protos/replay.proto | 7 +- 5 files changed, 333 insertions(+), 623 deletions(-) rename cloud/filestore/tools/testing/replay/lib/{request_replay.cpp => request_replay_fs.cpp} (55%) diff --git a/cloud/filestore/tools/testing/replay/lib/request.h b/cloud/filestore/tools/testing/replay/lib/request.h index ce99768233..456856154e 100644 --- a/cloud/filestore/tools/testing/replay/lib/request.h +++ b/cloud/filestore/tools/testing/replay/lib/request.h @@ -5,6 +5,7 @@ #include #include #include +//#include #include #include @@ -56,7 +57,14 @@ struct IRequestGenerator //////////////////////////////////////////////////////////////////////////////// -IRequestGeneratorPtr CreateReplayRequestGenerator( +IRequestGeneratorPtr CreateReplayRequestGeneratorFs( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers); + +IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( NProto::TReplaySpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay.cpp b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp similarity index 55% rename from cloud/filestore/tools/testing/replay/lib/request_replay.cpp rename to cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp index e9c28f3c64..a7fc655508 100644 --- a/cloud/filestore/tools/testing/replay/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp @@ -46,9 +46,9 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -class TReplayRequestGenerator final +class TReplayRequestGeneratorFs final : public IRequestGenerator - , public std::enable_shared_from_this + , public std::enable_shared_from_this { private: const NProto::TReplaySpec Spec; @@ -111,7 +111,7 @@ class TReplayRequestGenerator final TInstant Started; public: - TReplayRequestGenerator( + TReplayRequestGeneratorFs( NProto::TReplaySpec spec, ILoggingServicePtr logging, ISessionPtr session, @@ -124,18 +124,20 @@ class TReplayRequestGenerator final { Log = logging->CreateLog(Headers.GetClientId()); + if (Spec.GetReplayRoot().empty()) { + ythrow yexception() << "ReplayRoot is not defined"; + } + AsyncIO.Start(); NEventLog::TOptions options; options.FileName = Spec.GetFileName(); options.SetForceStrongOrdering(true); // need this? EventlogIterator = CreateIterator(options); - if (!Spec.GetReplayRoot().empty()) { - TFsPath(Spec.GetReplayRoot()).MkDirs(); - } + TFsPath(Spec.GetReplayRoot()).MkDirs(); } - ~TReplayRequestGenerator() + ~TReplayRequestGeneratorFs() { AsyncIO.Stop(); } @@ -171,11 +173,6 @@ class TReplayRequestGenerator final return 0; } - bool UseFs() - { - return !Spec.GetReplayRoot().empty(); - } - void Advance() { EventPtr = EventlogIterator->Next(); @@ -290,8 +287,8 @@ class TReplayRequestGenerator final } } } - STORAGE_DEBUG( - "Finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); + STORAGE_INFO( + "Log finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); return MakeFuture(TCompletedRequest(true)); } @@ -312,31 +309,25 @@ class TReplayRequestGenerator final } TGuard guard(StateLock); - if (UseFs()) { - const auto node = NodeIdMapped(r.GetNodeInfo().GetNodeId()); - if (!node) { - STORAGE_ERROR( - "access fail: " << " no node=" - << r.GetNodeInfo().GetNodeId()); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACCESS_NODE, - started, - MakeError(E_FAIL, "cancelled")}); - } + const auto node = NodeIdMapped(r.GetNodeInfo().GetNodeId()); - auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); - int res = access(fname.c_str(), R_OK); - STORAGE_DEBUG( - "access " << node << " <- " << r.GetNodeInfo().GetNodeId() - << " = " << res); - return MakeFuture( - TCompletedRequest{NProto::ACTION_ACCESS_NODE, started, {}}); + if (!node) { + STORAGE_ERROR( + "access fail: " << " no node=" << r.GetNodeInfo().GetNodeId()); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_FAIL, "cancelled")}); } - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACCESS_NODE, - started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + + auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); + int res = access(fname.c_str(), R_OK); + STORAGE_DEBUG( + "access " << node << " <- " << r.GetNodeInfo().GetNodeId() << " = " + << res); + return MakeFuture( + TCompletedRequest{NProto::ACTION_ACCESS_NODE, started, {}}); } // Recursive, no infinity loop check @@ -390,130 +381,95 @@ class TReplayRequestGenerator final TGuard guard(StateLock); auto started = TInstant::Now(); - if (UseFs()) { - TString relativePathName; - if (r.GetNodeInfo().GetNodeId()) { - if (auto path = PathByNode(r.GetNodeInfo().GetNodeId())) { - relativePathName = path; - } - } - - if (relativePathName.empty()) { - auto parentNode = - NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!parentNode) { - parentNode = NodeIdMapped( - KnownLogNodes[r.GetNodeInfo().GetParentNodeId()] - .ParentLog); - } - if (!parentNode && r.GetNodeInfo().GetParentNodeId() != - r.GetNodeInfo().GetNodeId()) - { - parentNode = CreateDirIfMissingByNodeLog( - r.GetNodeInfo().GetParentNodeId()); - } - - if (!parentNode) { - STORAGE_ERROR( - "create handle fail :" - << r.GetNodeInfo().GetHandle() - << " no parent=" << r.GetNodeInfo().GetParentNodeId()); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "cancelled")}); - } - - auto nodeName = r.GetNodeInfo().GetNodeName(); - if (nodeName.empty() && r.GetNodeInfo().GetNodeId() != - r.GetNodeInfo().GetParentNodeId()) - { - nodeName = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; - } - auto parentpath = PathByNode(parentNode); - - if (nodeName.empty() && - IsDir(Spec.GetReplayRoot() + parentpath)) - { - nodeName = - KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name; - } + TString relativePathName; + if (r.GetNodeInfo().GetNodeId()) { + if (auto path = PathByNode(r.GetNodeInfo().GetNodeId())) { + relativePathName = path; + } + } - relativePathName = parentpath + nodeName; + if (relativePathName.empty()) { + auto parentNode = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!parentNode) { + parentNode = NodeIdMapped( + KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].ParentLog); } - STORAGE_DEBUG( - "open " << relativePathName - << " handle=" << r.GetNodeInfo().GetHandle() - << " flags=" << r.GetNodeInfo().GetFlags() - << " mode=" << r.GetNodeInfo().GetMode() - << " node=" << r.GetNodeInfo().GetNodeId()); - - TFile fileHandle( - Spec.GetReplayRoot() + relativePathName, - OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - - if (!fileHandle.IsOpen()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "fail")}); + + if (!parentNode && r.GetNodeInfo().GetParentNodeId() != + r.GetNodeInfo().GetNodeId()) + { + parentNode = CreateDirIfMissingByNodeLog( + r.GetNodeInfo().GetParentNodeId()); } - const auto fh = fileHandle.GetHandle(); - if (!fh) { + + if (!parentNode) { + STORAGE_ERROR( + "create handle fail :" + << r.GetNodeInfo().GetHandle() + << " no parent=" << r.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, started, - MakeError(E_FAIL, "no filehandle")}); + MakeError(E_FAIL, "cancelled")}); } - OpenHandles[fh] = std::move(fileHandle); - HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; - const auto inode = - TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; - if (r.GetNodeInfo().GetNodeId()) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; + auto nodeName = r.GetNodeInfo().GetNodeName(); + if (nodeName.empty() && r.GetNodeInfo().GetNodeId() != + r.GetNodeInfo().GetParentNodeId()) + { + nodeName = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; + } + auto parentpath = PathByNode(parentNode); - NodePath[inode] = relativePathName; + if (nodeName.empty() && IsDir(Spec.GetReplayRoot() + parentpath)) { + nodeName = + KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name; } - STORAGE_DEBUG( - "open " << fh << "<-" << r.GetNodeInfo().GetHandle() - << " known handles=" << HandlesLogToActual.size() - << " opened=" << OpenHandles.size() - << " inode=" << inode); - - return MakeFuture( - TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); + + relativePathName = parentpath + nodeName; } + STORAGE_DEBUG( + "open " << relativePathName + << " handle=" << r.GetNodeInfo().GetHandle() + << " flags=" << r.GetNodeInfo().GetFlags() + << " mode=" << r.GetNodeInfo().GetMode() + << " node=" << r.GetNodeInfo().GetNodeId()); - auto request = CreateRequest(); - auto name = r.GetNodeInfo().GetNodeName(); + TFile fileHandle( + Spec.GetReplayRoot() + relativePathName, + OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{}); + if (!fileHandle.IsOpen()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "fail")}); + } + const auto fh = fileHandle.GetHandle(); + if (!fh) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "no filehandle")}); } - request->SetNodeId(node); - request->SetName(r.GetNodeInfo().GetNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); - request->SetMode(r.GetNodeInfo().GetMode()); - auto self = weak_from_this(); - return Session->CreateHandle(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr - ->HandleCreateHandle(future, name, started, r); - } + OpenHandles[fh] = std::move(fileHandle); + HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; + const auto inode = + TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; + if (r.GetNodeInfo().GetNodeId()) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "cancelled")}); - }); + NodePath[inode] = relativePathName; + } + STORAGE_DEBUG( + "open " << fh << "<-" << r.GetNodeInfo().GetHandle() + << " known handles=" << HandlesLogToActual.size() + << " opened=" << OpenHandles.size() << " inode=" << inode); + + return MakeFuture( + TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); } TFuture HandleCreateHandle( @@ -596,76 +552,41 @@ class TReplayRequestGenerator final } TGuard guard(StateLock); - if (UseFs()) { - const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); - if (!handle) { - STORAGE_WARN( - "read: no handle " - << r.GetRanges(0).GetHandle() - << " ranges size=" << r.GetRanges().size() - << " map size=" << HandlesLogToActual.size()); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_READ, - started, - MakeError(E_FAIL, "cancelled")}); - } - auto& fh = OpenHandles[handle]; - STORAGE_DEBUG( - "Read from " << handle << " fh.len=" << fh.GetLength() - << " fh.pos=" << fh.GetPosition()); - auto buffer = Acalloc(r.GetRanges().cbegin()->GetBytes()); - - fh.Reserve( - r.GetRanges().cbegin()->GetOffset() + - r.GetRanges().cbegin()->GetBytes()); - - TFileHandle FileHandle{fh.GetHandle()}; - - const auto future = AsyncIO.Read( - FileHandle, - {}, - r.GetRanges().cbegin()->GetBytes(), - r.GetRanges().cbegin()->GetOffset()); - FileHandle.Release(); - - return future.Apply( - [started]([[maybe_unused]] const auto& future) mutable { - return TCompletedRequest(NProto::ACTION_READ, started, {}); - }); - } - auto request = CreateRequest(); - const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); if (!handle) { - return MakeFuture(TCompletedRequest{}); + STORAGE_WARN( + "read: no handle " + << r.GetRanges(0).GetHandle() + << " ranges size=" << r.GetRanges().size() + << " map size=" << HandlesLogToActual.size()); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_FAIL, "cancelled")}); } - request->SetHandle(handle); - request->SetOffset(r.GetRanges().cbegin()->GetOffset()); - request->SetLength(r.GetRanges().cbegin()->GetBytes()); - - auto self = weak_from_this(); - return Session->ReadData(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - return TCompletedRequest{ - NProto::ACTION_READ, - started, - future.GetValue().GetError()}; - /* - if (auto ptr = self.lock()) { - return ptr->HandleRead( - future, - handleInfo, - started, - byteOffset); - } - */ - return TCompletedRequest{ - NProto::ACTION_READ, - started, - MakeError(E_FAIL, "cancelled")}; - }); + auto& fh = OpenHandles[handle]; + STORAGE_DEBUG( + "Read from " << handle << " fh.len=" << fh.GetLength() + << " fh.pos=" << fh.GetPosition()); + auto buffer = Acalloc(r.GetRanges().cbegin()->GetBytes()); + + fh.Reserve( + r.GetRanges().cbegin()->GetOffset() + + r.GetRanges().cbegin()->GetBytes()); + + TFileHandle FileHandle{fh.GetHandle()}; + + const auto future = AsyncIO.Read( + FileHandle, + {}, + r.GetRanges().cbegin()->GetBytes(), + r.GetRanges().cbegin()->GetOffset()); + FileHandle.Release(); + + return future.Apply( + [started]([[maybe_unused]] const auto& future) mutable + { return TCompletedRequest(NProto::ACTION_READ, started, {}); }); } TCompletedRequest HandleRead( @@ -711,85 +632,52 @@ class TReplayRequestGenerator final } TGuard guard(StateLock); - if (UseFs()) { - const auto logHandle = r.GetRanges(0).GetHandle(); - const auto handle = HandleIdMapped(logHandle); - if (!handle) { - return MakeFuture(TCompletedRequest( - NProto::ACTION_WRITE, - started, - MakeError( - E_CANCELLED, - TStringBuilder{} << "write cancelled: no handle =" - << logHandle))); // todo - } - const auto bytes = r.GetRanges(0).GetBytes(); - const auto offset = r.GetRanges(0).GetOffset(); - - TString buffer; - - if (Spec.GetWriteRandom()) { - buffer = NUnitTest::RandomString(bytes, logHandle); - } else if (Spec.GetWriteEmpty()) { - buffer = TString{bytes, ' '}; - } else { - buffer = MakeBuffer( - bytes, - offset, - TStringBuilder{} << "handle=" << logHandle - << " node=" << r.GetNodeInfo().GetNodeId() - << " bytes=" << bytes - << " offset=" << offset); - } - - auto& fh = OpenHandles[handle]; - - STORAGE_DEBUG( - "Write to " << handle << " fh.length=" << fh.GetLength() - << " fh.pos=" << fh.GetPosition()); - // TODO TEST USE AFTER FREE on buffer - TFileHandle FileHandle{fh.GetHandle()}; - const auto writeFuture = AsyncIO.Write( - // fh, - FileHandle, - buffer.data(), - bytes, - offset); - FileHandle.Release(); - return writeFuture.Apply( - [started]([[maybe_unused]] const auto& future) mutable { - return TCompletedRequest(NProto::ACTION_WRITE, started, {}); - }); - } - - auto request = CreateRequest(); - const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); - + const auto logHandle = r.GetRanges(0).GetHandle(); + const auto handle = HandleIdMapped(logHandle); if (!handle) { - return MakeFuture(TCompletedRequest{}); + return MakeFuture(TCompletedRequest( + NProto::ACTION_WRITE, + started, + MakeError( + E_CANCELLED, + TStringBuilder{} << "write cancelled: no handle =" + << logHandle))); // todo } - request->SetHandle(handle); + const auto bytes = r.GetRanges(0).GetBytes(); + const auto offset = r.GetRanges(0).GetOffset(); - request->SetOffset(r.GetRanges(0).GetOffset()); - auto bytes = r.GetRanges(0).GetBytes(); - auto buffer = NUnitTest::RandomString(bytes); + TString buffer; - *request->MutableBuffer() = std::move(buffer); + if (Spec.GetWriteRandom()) { + buffer = NUnitTest::RandomString(bytes, logHandle); + } else if (Spec.GetWriteEmpty()) { + buffer = TString{bytes, ' '}; + } else { + buffer = MakeBuffer( + bytes, + offset, + TStringBuilder{} << "handle=" << logHandle + << " node=" << r.GetNodeInfo().GetNodeId() + << " bytes=" << bytes << " offset=" << offset); + } - auto self = weak_from_this(); - return Session->WriteData(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleWrite(future, started); - } + auto& fh = OpenHandles[handle]; - return TCompletedRequest{ - NProto::ACTION_WRITE, - started, - MakeError(E_FAIL, "cancelled")}; - }); + STORAGE_DEBUG( + "Write to " << handle << " fh.length=" << fh.GetLength() + << " fh.pos=" << fh.GetPosition()); + // TODO TEST USE AFTER FREE on buffer + TFileHandle FileHandle{fh.GetHandle()}; + const auto writeFuture = AsyncIO.Write( + // fh, + FileHandle, + buffer.data(), + bytes, + offset); + FileHandle.Release(); + return writeFuture.Apply( + [started]([[maybe_unused]] const auto& future) mutable + { return TCompletedRequest(NProto::ACTION_WRITE, started, {}); }); } TCompletedRequest HandleWrite( @@ -845,126 +733,66 @@ class TReplayRequestGenerator final TGuard guard(StateLock); - if (UseFs()) { - auto parentNode = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - - parentNode = CreateDirIfMissingByNodeLog( - r.GetNodeInfo().GetNewParentNodeId()); - - auto fullName = Spec.GetReplayRoot() + "/" + - PathByNode(parentNode) + - r.GetNodeInfo().GetNewNodeName(); - - ui64 nodeid = 0; - bool isDir = false; - switch (r.GetNodeInfo().GetType()) { - case NProto::E_REGULAR_NODE: { - // TODO: transform r.GetNodeInfo().GetMode() to correct open - // mode - TFileHandle fh( - fullName, - OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - fh.Reserve(r.GetNodeInfo().GetSize()); - if (fh) { - nodeid = TFileStat{fh}.INode; - } else { - nodeid = TFileStat{fullName}.INode; - } - } break; - case NProto::E_DIRECTORY_NODE: { - isDir = true; - nodeid = MakeDirectoryRecursive(fullName); - } break; - case NProto::E_LINK_NODE: - // TODO: NFs::HardLink(const TString &existingPath, const - // TString &newPath) NFs::SymLink(const TString &targetPath, - // const TString &linkPath) - break; - case NProto::E_SOCK_NODE: - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + auto parentNode = NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - case NProto::E_INVALID_NODE: - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); - } + parentNode = + CreateDirIfMissingByNodeLog(r.GetNodeInfo().GetNewParentNodeId()); - if (!nodeid) { - nodeid = TFileStat{fullName}.INode; - } - - // CreateIfMissing(PathByNode()) - if (nodeid) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = nodeid; - NodePath[nodeid] = PathByNode(parentNode) + - r.GetNodeInfo().GetNewNodeName() + - (isDir ? "/" : ""); - } - - return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); - } - auto request = CreateRequest(); - - const auto parentNode = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - if (!parentNode) { - return MakeFuture(TCompletedRequest{}); - } - request->SetNodeId(parentNode); - auto name = r.GetNodeInfo().GetNewNodeName(); - request->SetName(r.GetNodeInfo().GetNewNodeName()); - - // request->SetGid(); - // request->SetUid(); + auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + + r.GetNodeInfo().GetNewNodeName(); + ui64 nodeid = 0; + bool isDir = false; switch (r.GetNodeInfo().GetType()) { - case NProto::E_REGULAR_NODE: - request->MutableFile()->SetMode(r.GetNodeInfo().GetMode()); - break; - case NProto::E_DIRECTORY_NODE: - request->MutableDirectory()->SetMode(r.GetNodeInfo().GetMode()); - break; + case NProto::E_REGULAR_NODE: { + // TODO: transform r.GetNodeInfo().GetMode() to correct open + // mode + TFileHandle fh( + fullName, + OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); + fh.Reserve(r.GetNodeInfo().GetSize()); + if (fh) { + nodeid = TFileStat{fh}.INode; + } else { + nodeid = TFileStat{fullName}.INode; + } + } break; + case NProto::E_DIRECTORY_NODE: { + isDir = true; + nodeid = MakeDirectoryRecursive(fullName); + } break; case NProto::E_LINK_NODE: - // TODO: - // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); - // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); - return MakeFuture(TCompletedRequest{}); - break; - case NProto::E_SYMLINK_NODE: - // TODO: - // request->MutableSymlink()->SetTargetPath(); + // TODO: NFs::HardLink(const TString &existingPath, const + // TString &newPath) NFs::SymLink(const TString &targetPath, + // const TString &linkPath) break; case NProto::E_SOCK_NODE: - request->MutableSocket()->SetMode(r.GetNodeInfo().GetMode()); - break; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + case NProto::E_INVALID_NODE: - return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); - // Do not create files with invalid - // type - too hard to delete them - break; + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); } - auto self = weak_from_this(); - return Session->CreateNode(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateNode(future, name, started, r); - } + if (!nodeid) { + nodeid = TFileStat{fullName}.INode; + } - return TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + // CreateIfMissing(PathByNode()) + if (nodeid) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = nodeid; + NodePath[nodeid] = PathByNode(parentNode) + + r.GetNodeInfo().GetNewNodeName() + + (isDir ? "/" : ""); + } + + return MakeFuture( + TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); } TCompletedRequest HandleCreateNode( @@ -1014,62 +842,23 @@ class TReplayRequestGenerator final TGuard guard(StateLock); - if (UseFs()) { - const auto parentnodeid = - NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - - auto fullName = Spec.GetReplayRoot() + PathByNode(parentnodeid) + - r.GetNodeInfo().GetNodeName(); - - const auto newparentnodeid = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - - auto newFullName = Spec.GetReplayRoot() + - PathByNode(newparentnodeid) + - r.GetNodeInfo().GetNewNodeName(); - const auto renameres = NFs::Rename(fullName, newFullName); - STORAGE_DEBUG( - "rename " << fullName << " => " << newFullName << " : " - << renameres); - return MakeFuture( - TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); - } + const auto parentnodeid = + NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - auto request = CreateRequest(); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - started, - MakeError( - E_FAIL, - TStringBuilder{} << "Log node " - << r.GetNodeInfo().GetParentNodeId() - << "not found in mappiong")}); - } - request->SetNodeId(node); - request->SetName(r.GetNodeInfo().GetNodeName()); - request->SetNewParentId( - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId())); - request->SetNewName(r.GetNodeInfo().GetNewNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); - - auto self = weak_from_this(); - return Session->RenameNode(CreateCallContext(), std::move(request)) - .Apply( - [= + auto fullName = Spec.GetReplayRoot() + PathByNode(parentnodeid) + + r.GetNodeInfo().GetNodeName(); - ](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleRenameNode(future, started, r); - } + const auto newparentnodeid = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); - return TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + auto newFullName = Spec.GetReplayRoot() + PathByNode(newparentnodeid) + + r.GetNodeInfo().GetNewNodeName(); + const auto renameres = NFs::Rename(fullName, newFullName); + STORAGE_DEBUG( + "rename " << fullName << " => " << newFullName << " : " + << renameres); + return MakeFuture( + TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); } TCompletedRequest HandleRenameNode( @@ -1108,53 +897,27 @@ class TReplayRequestGenerator final TGuard guard(StateLock); - if (UseFs()) { - const auto parentNodeId = - NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!parentNodeId) { - STORAGE_WARN( - "unlink : no parent orig=" - << r.GetNodeInfo().GetParentNodeId()); - return MakeFuture(TCompletedRequest( - NProto::ACTION_REMOVE_NODE, - started, - MakeError(E_CANCELLED, "cancelled"))); - } - const auto fullName = Spec.GetReplayRoot() + "/" + - PathByNode(parentNodeId) + - r.GetNodeInfo().GetNodeName(); - const auto unlinkres = NFs::Remove(fullName); - STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); - // TODO : - // NodesLogToActual.erase(...) - // NodePath.erase(...) - return MakeFuture( - TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); - } - - auto name = r.GetNodeInfo().GetNodeName(); - auto request = CreateRequest(); - request->SetName(name); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{}); + const auto parentNodeId = + NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!parentNodeId) { + STORAGE_WARN( + "unlink : no parent orig=" + << r.GetNodeInfo().GetParentNodeId()); + return MakeFuture(TCompletedRequest( + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_CANCELLED, "cancelled"))); } - request->SetNodeId(node); - auto self = weak_from_this(); - return Session->UnlinkNode(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleUnlinkNode(future, name, started); - } - - return TCompletedRequest{ - NProto::ACTION_REMOVE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + const auto fullName = Spec.GetReplayRoot() + "/" + + PathByNode(parentNodeId) + + r.GetNodeInfo().GetNodeName(); + const auto unlinkres = NFs::Remove(fullName); + STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); + // TODO : + // NodesLogToActual.erase(...) + // NodePath.erase(...) + return MakeFuture( + TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); } TCompletedRequest HandleUnlinkNode( @@ -1185,65 +948,34 @@ class TReplayRequestGenerator final TGuard guard(StateLock); auto started = TInstant::Now(); - if (UseFs()) { - const auto handleid = HandleIdMapped(r.GetNodeInfo().GetHandle()); - - const auto& it = OpenHandles.find(handleid); - if (it == OpenHandles.end()) { - return MakeFuture(TCompletedRequest( - NProto::ACTION_DESTROY_HANDLE, - started, - MakeError( - E_CANCELLED, - TStringBuilder{} << "close " << handleid << " <- " - << r.GetNodeInfo().GetHandle() - << " fail: not found in " - << OpenHandles.size()))); - } - - auto& fhandle = it->second; - const auto len = fhandle.GetLength(); - const auto pos = fhandle.GetPosition(); - fhandle.Close(); - OpenHandles.erase(handleid); - HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); - STORAGE_DEBUG( - "Close " << handleid << " orig=" << r.GetNodeInfo().GetHandle() - << " pos=" << pos << " len=" << len - << " open map size=" << OpenHandles.size() - << " map size=" << HandlesLogToActual.size()); - return MakeFuture( - TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, started, {})); - } - - auto name = r.GetNodeInfo().GetNodeName(); + const auto handleid = HandleIdMapped(r.GetNodeInfo().GetHandle()); - auto request = CreateRequest(); - - const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); - if (!handle) { - return MakeFuture(TCompletedRequest{}); + const auto& it = OpenHandles.find(handleid); + if (it == OpenHandles.end()) { + return MakeFuture(TCompletedRequest( + NProto::ACTION_DESTROY_HANDLE, + started, + MakeError( + E_CANCELLED, + TStringBuilder{} << "close " << handleid << " <- " + << r.GetNodeInfo().GetHandle() + << " fail: not found in " + << OpenHandles.size()))); } - HandlesLogToActual.erase(handle); - - request->SetHandle(handle); - - auto self = weak_from_this(); - return Session->DestroyHandle(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleDestroyHandle(name, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_DESTROY_HANDLE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + auto& fhandle = it->second; + const auto len = fhandle.GetLength(); + const auto pos = fhandle.GetPosition(); + fhandle.Close(); + OpenHandles.erase(handleid); + HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); + STORAGE_DEBUG( + "Close " << handleid << " orig=" << r.GetNodeInfo().GetHandle() + << " pos=" << pos << " len=" << len + << " open map size=" << OpenHandles.size() + << " map size=" << HandlesLogToActual.size()); + return MakeFuture( + TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, started, {})); } TFuture DoGetNodeAttr( @@ -1266,60 +998,32 @@ class TReplayRequestGenerator final // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} - if (UseFs()) { - if (r.GetNodeInfo().GetNodeName()) { - KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name = - r.GetNodeInfo().GetNodeName(); - } - if (r.GetNodeInfo().GetParentNodeId() && - r.GetNodeInfo().GetParentNodeId() != - r.GetNodeInfo().GetNodeId()) - { - KnownLogNodes[r.GetNodeInfo().GetNodeId()].ParentLog = - r.GetNodeInfo().GetParentNodeId(); - } - - // TODO: can create and truncate to size here missing files + if (r.GetNodeInfo().GetNodeName()) { + KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name = + r.GetNodeInfo().GetNodeName(); + } + if (r.GetNodeInfo().GetParentNodeId() && + r.GetNodeInfo().GetParentNodeId() != r.GetNodeInfo().GetNodeId()) + { + KnownLogNodes[r.GetNodeInfo().GetNodeId()].ParentLog = + r.GetNodeInfo().GetParentNodeId(); + } - const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + // TODO: can create and truncate to size here missing files - if (!nodeid) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError(E_CANCELLED, "cancelled")}); - } + const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); - auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); - [[maybe_unused]] const auto stat = TFileStat{fname}; - return MakeFuture( - TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); - } - auto request = CreateRequest(); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{}); + if (!nodeid) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_CANCELLED, "cancelled")}); } - request->SetNodeId(node); - auto name = r.GetNodeInfo().GetNodeName(); - request->SetName(r.GetNodeInfo().GetNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); - auto self = weak_from_this(); - STORAGE_DEBUG("GetNodeAttr client started"); - return Session->GetNodeAttr(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleGetNodeAttr(name, future, started); - } - return TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); + [[maybe_unused]] const auto stat = TFileStat{fname}; + return MakeFuture( + TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); } TCompletedRequest HandleGetNodeAttr( @@ -1534,14 +1238,14 @@ class TReplayRequestGenerator final //////////////////////////////////////////////////////////////////////////////// -IRequestGeneratorPtr CreateReplayRequestGenerator( +IRequestGeneratorPtr CreateReplayRequestGeneratorFs( NProto::TReplaySpec spec, ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, NProto::THeaders headers) { - return std::make_shared( + return std::make_shared( std::move(spec), std::move(logging), std::move(session), diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp index d9d4915a84..252c1e9788 100644 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -2,7 +2,6 @@ #include "client.h" #include "context.h" -#include "dump.h" #include "request.h" #include @@ -261,7 +260,7 @@ class TLoadTest final TString SessionId; ui32 MaxIoDepth = 64; - i32 CurrentIoDepth = 0; + ui32 CurrentIoDepth = 0; TDuration MaxDuration; TInstant StartTs; @@ -320,7 +319,7 @@ class TLoadTest final { if (LoadEnabled && !ShouldStop()) { while (SendNextRequest()) { - //DUMP("rb", RequestsSent); + // DUMP("rb", RequestsSent); ++RequestsSent; STORAGE_DEBUG( "%s request sent: %s %lu %lu", @@ -329,8 +328,6 @@ class TLoadTest final SeqNo, RequestsSent); } - DUMP("fin"); -//PrintBackTrace(); } } @@ -455,12 +452,12 @@ class TLoadTest final LastReportTs = TInstant::Now(); for (;;) { - Event.WaitI(); + Event.WaitI(); bool cont = true; while (auto maybeCmd = Commands.Dequeue()) { auto& cmd = *maybeCmd; if (cont = HandleCommand(cmd); !cont) { - break; + break; } } if (ShouldStop() && !ShutdownFlag) { @@ -526,13 +523,19 @@ class TLoadTest final headers.SetClientId(Config.GetName()); headers.SetSessionId(SessionId); - DUMP(Config); - switch (Config.GetSpecsCase()) { - case NProto::TLoadTest::kReplaySpec: - // DUMP("repl", Config.GetReplaySpec().GetFileName()); - RequestGenerator = CreateReplayRequestGenerator( - Config.GetReplaySpec(), + case NProto::TLoadTest::kReplayFsSpec: + RequestGenerator = CreateReplayRequestGeneratorFs( + Config.GetReplayFsSpec(), + Logging, + Session, + FileSystemId, + headers); + break; + + case NProto::TLoadTest::kReplayGrpcSpec: + RequestGenerator = CreateReplayRequestGeneratorGRPC( + Config.GetReplayGrpcSpec(), Logging, Session, FileSystemId, @@ -574,11 +577,9 @@ class TLoadTest final bool SendNextRequest() { - if (LimitsReached() || - //(MaxIoDepth && CurrentIoDepth >= MaxIoDepth) || + if (LimitsReached() || (MaxIoDepth && CurrentIoDepth >= MaxIoDepth) || !RequestGenerator->HasNextRequest() || ShouldStop()) { - DUMP("nop"); return false; } @@ -678,7 +679,6 @@ class TLoadTest final void TeardownTest(TTeardown& cmd) { - DUMP("teardown", ShutdownFlag, CurrentIoDepth); if (ShutdownFlag) { if (cmd.Complete.Initialized()) { cmd.Complete.SetValue(true); @@ -688,11 +688,9 @@ class TLoadTest final STORAGE_INFO("%s tear down", MakeTestTag().c_str()); while (CurrentIoDepth > 0) { - DUMP("td io", CurrentIoDepth); Sleep(TDuration::Seconds(1)); ProcessCompletedRequests(); } - DUMP("tdok"); auto ctx = MakeIntrusive(); auto result = Session->DestroySession(); diff --git a/cloud/filestore/tools/testing/replay/lib/ya.make b/cloud/filestore/tools/testing/replay/lib/ya.make index c24c4d3757..fab1d4ac2e 100644 --- a/cloud/filestore/tools/testing/replay/lib/ya.make +++ b/cloud/filestore/tools/testing/replay/lib/ya.make @@ -4,7 +4,8 @@ SRCS( client.cpp context.h executor.cpp - request_replay.cpp + request_replay_fs.cpp + request_replay_grpc.cpp test.cpp ) diff --git a/cloud/filestore/tools/testing/replay/protos/replay.proto b/cloud/filestore/tools/testing/replay/protos/replay.proto index 2db7257abb..09553755a3 100644 --- a/cloud/filestore/tools/testing/replay/protos/replay.proto +++ b/cloud/filestore/tools/testing/replay/protos/replay.proto @@ -32,7 +32,7 @@ message TReplaySpec string FileName = 1; //bool ValidationEnabled = 6; //bool Sequential = 7; - string ReplayRoot = 8; + string ReplayRoot = 8; // not used in grpc bool NoRead = 9; bool NoWrite = 10; bool Prepare = 11; // Only write files needed for reading @@ -62,9 +62,8 @@ message TLoadTest oneof Specs { - //TIndexLoadSpec IndexLoadSpec = 4; - //TDataLoadSpec DataLoadSpec = 5; - TReplaySpec ReplaySpec = 17; + TReplaySpec ReplayFsSpec = 17; + TReplaySpec ReplayGrpcSpec = 18; } uint32 IODepth = 10; From 99c051b57b5d689899fe120e5209fbe7eb55f1fe Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 24 Sep 2024 15:15:17 +0000 Subject: [PATCH 06/44] wip --- cloud/filestore/tools/testing/replay/lib/test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp index 252c1e9788..1a644f6faa 100644 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -585,7 +585,7 @@ class TLoadTest final auto self = weak_from_this(); ++CurrentIoDepth; - RequestGenerator->ExecuteNextRequest().Apply( + const auto future = RequestGenerator->ExecuteNextRequest().Apply( [=](const TFuture& future) { if (auto ptr = self.lock()) { @@ -594,7 +594,7 @@ class TLoadTest final }); if (RequestGenerator->InstantProcessQueue()) { - ProcessCompletedRequests(); + future.GetValueSync(); } return true; From 2b76fdc752095918ec5ce47999379d62daafeacd Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 24 Sep 2024 18:03:12 +0000 Subject: [PATCH 07/44] wip (listnodes,...) --- .../testing/replay/lib/request_replay_fs.cpp | 297 ++++-------------- .../tools/testing/replay/lib/test.cpp | 4 +- .../tools/testing/replay/lib/ya.make | 1 + .../tools/testing/replay/protos/replay.proto | 1 + 4 files changed, 71 insertions(+), 232 deletions(-) diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp index a7fc655508..b63a5cf12b 100644 --- a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp @@ -20,6 +20,7 @@ compare log and actual result ( S_OK E_FS_NOENT ...) #include #include +#include #include #include #include @@ -68,8 +69,6 @@ class TReplayRequestGeneratorFs final using TNodeLog = ui64; using TNodeLocal = ui64; - ui64 InitialFileSize = 0; - std::atomic LastRequestId = 0; THolder EventlogIterator; @@ -142,6 +141,30 @@ class TReplayRequestGeneratorFs final AsyncIO.Stop(); } + template + std::shared_ptr CreateRequest() + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->CopyFrom(Headers); + + return request; + } + + template + void CheckResponse(const T& response) + { + if (HasError(response)) { + throw TServiceError(response.GetError()); + } + } + + TIntrusivePtr CreateCallContext() + { + return MakeIntrusive( + LastRequestId.fetch_add(1, std::memory_order_relaxed)); + } + bool InstantProcessQueue() override { return true; @@ -270,6 +293,8 @@ class TReplayRequestGeneratorFs final case EFileStoreRequest::AccessNode: return DoAccessNode(request); + case EFileStoreRequest::ListNodes: + return DoListNodes(request); case EFileStoreRequest::ReadBlob: case EFileStoreRequest::WriteBlob: @@ -464,69 +489,13 @@ class TReplayRequestGeneratorFs final NodePath[inode] = relativePathName; } STORAGE_DEBUG( - "open " << fh << "<-" << r.GetNodeInfo().GetHandle() - << " known handles=" << HandlesLogToActual.size() - << " opened=" << OpenHandles.size() << " inode=" << inode); + "open " << fh << "<-" << r.GetNodeInfo().GetHandle() << " inode=" + << inode << " known handles=" << HandlesLogToActual.size() + << " opened=" << OpenHandles.size()); return MakeFuture( TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); } - - TFuture HandleCreateHandle( - const TFuture& future, - const TString& name, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - auto handle = response.GetHandle(); - with_lock (StateLock) { - HandlesLogToActual[r.GetNodeInfo().GetHandle()] = handle; - } - - NThreading::TFuture setAttr; - if (InitialFileSize) { - static const int flags = - ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetNodeId(response.GetNodeAttr().GetId()); - request->SetFlags(flags); - request->MutableUpdate()->SetSize(InitialFileSize); - - setAttr = Session->SetNodeAttr( - CreateCallContext(), - std::move(request)); - } else { - setAttr = - NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); - } - - return setAttr.Apply( - [=]([[maybe_unused]] const TFuture< - NProto::TSetNodeAttrResponse>& f) - { - return MakeFuture(TCompletedRequest{}); - // return HandleResizeAfterCreateHandle(f, name, started); - }); - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "create handle for %s has failed: %s", - name.Quote().c_str(), - FormatError(error).c_str()); - - return NThreading::MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - error}); - } - } - static constexpr ui32 BlockSize = 4_KB; static std::shared_ptr Acalloc(ui64 dataSize) { @@ -589,25 +558,6 @@ class TReplayRequestGeneratorFs final { return TCompletedRequest(NProto::ACTION_READ, started, {}); }); } - TCompletedRequest HandleRead( - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - return {NProto::ACTION_READ, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "read for %s has failed: ", - // handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_READ, started, error}; - } - } - TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) { TStringBuilder ret; @@ -680,27 +630,6 @@ class TReplayRequestGeneratorFs final { return TCompletedRequest(NProto::ACTION_WRITE, started, {}); }); } - TCompletedRequest HandleWrite( - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_WRITE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - /* - STORAGE_ERROR( - "write on %s has failed: %s", - handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - */ - return {NProto::ACTION_WRITE, started, error}; - } - } - TString PathByNode(TNodeLocal nodeid) { if (const auto& it = NodePath.find(nodeid); it != NodePath.end()) { @@ -794,36 +723,6 @@ class TReplayRequestGeneratorFs final return MakeFuture( TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); } - - TCompletedRequest HandleCreateNode( - const TFuture& future, - const TString& name, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) - { - TGuard guard(StateLock); - - try { - auto response = future.GetValue(); - CheckResponse(response); - // Nodes[name] = TNode{name, response.GetNode()}; - if (response.GetNode().GetId()) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = - response.GetNode().GetId(); - } - - return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "create node %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_NODE, started, error}; - } - } - TFuture DoRenameNode( // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, @@ -861,26 +760,6 @@ class TReplayRequestGeneratorFs final TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); } - TCompletedRequest HandleRenameNode( - const TFuture& future, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) - { - TGuard guard(StateLock); - try { - auto response = future.GetValue(); - CheckResponse(response); - return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "rename node %s has failed: %s", - r.GetNodeInfo().GetNodeName().c_str(), - FormatError(error).c_str()); - return {NProto::ACTION_RENAME_NODE, started, error}; - } - } - TFuture DoUnlinkNode( NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { @@ -920,26 +799,6 @@ class TReplayRequestGeneratorFs final TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); } - TCompletedRequest HandleUnlinkNode( - const TFuture& future, - const TString& name, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "unlink for %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_REMOVE_NODE, started, error}; - } - } TFuture DoDestroyHandle( NCloud::NFileStore::NProto::TProfileLogRequestInfo r) { @@ -1026,49 +885,6 @@ class TReplayRequestGeneratorFs final TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); } - TCompletedRequest HandleGetNodeAttr( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - STORAGE_DEBUG("GetNodeAttr client completed"); - CheckResponse(response); - TGuard guard(StateLock); - return {NProto::ACTION_GET_NODE_ATTR, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "get node attr %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_GET_NODE_ATTR, started, {}}; - } - } - - TCompletedRequest HandleDestroyHandle( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - auto response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_DESTROY_HANDLE, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "destroy handle %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_DESTROY_HANDLE, started, error}; - } - } - TFuture DoAcquireLock() { TGuard guard(StateLock); @@ -1209,28 +1025,47 @@ class TReplayRequestGeneratorFs final } } - template - std::shared_ptr CreateRequest() + TFuture DoListNodes( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { - auto request = std::make_shared(); - request->SetFileSystemId(FileSystemId); - request->MutableHeaders()->CopyFrom(Headers); + // json={"TimestampMcs":1726615510721016,"DurationMcs":3329,"RequestType":30,"ErrorCode":0,"NodeInfo":{"NodeId":164,"Size":10}} - return request; - } + auto started = TInstant::Now(); + TGuard guard(StateLock); - template - void CheckResponse(const T& response) - { - if (HasError(response)) { - throw TServiceError(response.GetError()); + const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + if (!nodeid) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_LIST_NODES, + started, + MakeError( + E_CANCELLED, + TStringBuilder{} << "Node not found in mapping" + << nodeid)}); } - } - TIntrusivePtr CreateCallContext() - { - return MakeIntrusive( - LastRequestId.fetch_add(1, std::memory_order_relaxed)); + // if (!Spec.GetNoWrite()) { + // CreateDirIfMissingByNodeLog(r.GetNodeInfo().GetNodeId()); + // } + + auto path = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); + TFileHandle dir{path, RdOnly}; + if (!dir.IsOpen()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_LIST_NODES, + started, + MakeError(E_CANCELLED, "cancelled")}); + } + const auto dirs = NLowLevel::ListDirAt(dir, true); + if (r.GetNodeInfo().GetSize() != dirs.size()) { + STORAGE_DEBUG( + "Dir size differs " << path + << " log=" << r.GetNodeInfo().GetSize() + << " local=" << dirs.size()); + } + + return MakeFuture( + TCompletedRequest(NProto::ACTION_LIST_NODES, started, {})); } }; diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp index 1a644f6faa..a4238cc00c 100644 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -594,7 +594,9 @@ class TLoadTest final }); if (RequestGenerator->InstantProcessQueue()) { - future.GetValueSync(); + if (future.HasValue()) { + ProcessCompletedRequests(); + } } return true; diff --git a/cloud/filestore/tools/testing/replay/lib/ya.make b/cloud/filestore/tools/testing/replay/lib/ya.make index fab1d4ac2e..ea59021d8d 100644 --- a/cloud/filestore/tools/testing/replay/lib/ya.make +++ b/cloud/filestore/tools/testing/replay/lib/ya.make @@ -25,6 +25,7 @@ PEERDIR( library/cpp/eventlog cloud/filestore/libs/diagnostics/events + cloud/filestore/libs/service_local cloud/filestore/tools/analytics/libs/event-log library/cpp/testing/unittest diff --git a/cloud/filestore/tools/testing/replay/protos/replay.proto b/cloud/filestore/tools/testing/replay/protos/replay.proto index 09553755a3..c174cd5eae 100644 --- a/cloud/filestore/tools/testing/replay/protos/replay.proto +++ b/cloud/filestore/tools/testing/replay/protos/replay.proto @@ -20,6 +20,7 @@ enum EAction ACTION_RELEASE_LOCK = 6; ACTION_GET_NODE_ATTR = 7; ACTION_ACCESS_NODE = 8; + ACTION_LIST_NODES = 9; // DATA ACTION_WRITE = 101; From a79bd290e9b313c1dddb4326bff7d335650abf9b Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 24 Sep 2024 19:53:20 +0000 Subject: [PATCH 08/44] correct structure --- .../tools/testing/replay/lib/request.h | 5 +- .../testing/replay/lib/request_replay_fs.cpp | 125 ++++++++++++------ .../tools/testing/replay/lib/test.cpp | 7 +- .../tools/testing/replay/protos/replay.proto | 1 + 4 files changed, 96 insertions(+), 42 deletions(-) diff --git a/cloud/filestore/tools/testing/replay/lib/request.h b/cloud/filestore/tools/testing/replay/lib/request.h index 456856154e..ecad5a7d97 100644 --- a/cloud/filestore/tools/testing/replay/lib/request.h +++ b/cloud/filestore/tools/testing/replay/lib/request.h @@ -5,7 +5,6 @@ #include #include #include -//#include #include #include @@ -53,6 +52,10 @@ struct IRequestGenerator { return false; } + virtual bool FailOnError() + { + return true; + } }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp index b63a5cf12b..015c29448b 100644 --- a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp @@ -4,13 +4,14 @@ create file/dir modes create handle modes (now rw) compare log and actual result ( S_OK E_FS_NOENT ...) - +read/write with multiranges (now only first processed) */ #include "request.h" #include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" #include "cloud/filestore/tools/analytics/libs/event-log/dump.h" +#include "dump.h" #include "library/cpp/aio/aio.h" #include "library/cpp/eventlog/iterator.h" #include "library/cpp/testing/unittest/registar.h" @@ -170,6 +171,11 @@ class TReplayRequestGeneratorFs final return true; } + bool FailOnError() override + { + return false; + } + TNodeLocal NodeIdMapped(const TNodeLog id) { if (const auto it = NodesLogToLocal.find(id); @@ -239,7 +245,6 @@ class TReplayRequestGeneratorFs final for (; EventMessageNumber > 0;) { auto request = EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - STORAGE_DEBUG("message json=" << request.AsJSON()); auto timediff = (request.GetTimestampMcs() - TimestampMcs) * Spec.GetTimeScale(); TimestampMcs = request.GetTimestampMcs(); @@ -265,8 +270,8 @@ class TReplayRequestGeneratorFs final << EventMessageNumber << " typename=" << request.GetTypeName() << " type=" << request.GetRequestType() << " " - << RequestName(request.GetRequestType())); - + << " name=" << RequestName(request.GetRequestType()) + << " json=" << request.AsJSON()); { const auto& action = request.GetRequestType(); switch (static_cast(action)) { @@ -286,16 +291,17 @@ class TReplayRequestGeneratorFs final return DoDestroyHandle(request); case EFileStoreRequest::GetNodeAttr: return DoGetNodeAttr(request); - case EFileStoreRequest::AcquireLock: - return DoAcquireLock(); - case EFileStoreRequest::ReleaseLock: - return DoReleaseLock(); - case EFileStoreRequest::AccessNode: return DoAccessNode(request); case EFileStoreRequest::ListNodes: return DoListNodes(request); + // TODO: + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(); + case EFileStoreRequest::ReadBlob: case EFileStoreRequest::WriteBlob: case EFileStoreRequest::GenerateBlobIds: @@ -395,6 +401,22 @@ class TReplayRequestGeneratorFs final } } + static EOpenModeFlag FileOpenFlags(ui32 flags, EOpenModeFlag init = {}) + { + auto systemflags = HandleFlagsToSystem(flags); + ui32 value{init}; + + if (systemflags & O_RDWR) { + value |= EOpenModeFlag::RdWr | EOpenModeFlag::OpenAlways; + } else if (systemflags & O_WRONLY) { + value |= EOpenModeFlag::WrOnly | EOpenModeFlag::OpenAlways; + } else if (systemflags & O_RDONLY) { + value |= EOpenModeFlag::RdOnly; + } + + return static_cast(value); + } + TFuture DoCreateHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) { @@ -457,42 +479,56 @@ class TReplayRequestGeneratorFs final STORAGE_DEBUG( "open " << relativePathName << " handle=" << r.GetNodeInfo().GetHandle() - << " flags=" << r.GetNodeInfo().GetFlags() + << " flags=" << r.GetNodeInfo().GetFlags() << " " + << HandleFlagsToString(r.GetNodeInfo().GetFlags()) << " mode=" << r.GetNodeInfo().GetMode() << " node=" << r.GetNodeInfo().GetNodeId()); - TFile fileHandle( - Spec.GetReplayRoot() + relativePathName, - OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); + try { + TFile fileHandle( + Spec.GetReplayRoot() + relativePathName, + // OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr) + FileOpenFlags( + r.GetNodeInfo().GetFlags(), + Spec.GetNoWrite() ? OpenExisting + : Spec.GetCreateOnRead() ? OpenAlways + : OpenExisting)); + + if (!fileHandle.IsOpen()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "fail")}); + } + const auto fh = fileHandle.GetHandle(); + if (!fh) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "no filehandle")}); + } - if (!fileHandle.IsOpen()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "fail")}); - } - const auto fh = fileHandle.GetHandle(); - if (!fh) { + OpenHandles[fh] = std::move(fileHandle); + HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; + const auto inode = + TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; + if (r.GetNodeInfo().GetNodeId()) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; + + NodePath[inode] = relativePathName; + } + STORAGE_DEBUG( + "open " << fh << "<-" << r.GetNodeInfo().GetHandle() + << " inode=" << inode + << " known handles=" << HandlesLogToActual.size() + << " opened=" << OpenHandles.size()); + } catch (const TFileError& error) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, started, - MakeError(E_FAIL, "no filehandle")}); + MakeError(E_FAIL, error.what())}); } - OpenHandles[fh] = std::move(fileHandle); - HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; - const auto inode = - TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; - if (r.GetNodeInfo().GetNodeId()) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; - - NodePath[inode] = relativePathName; - } - STORAGE_DEBUG( - "open " << fh << "<-" << r.GetNodeInfo().GetHandle() << " inode=" - << inode << " known handles=" << HandlesLogToActual.size() - << " opened=" << OpenHandles.size()); - return MakeFuture( TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); } @@ -752,10 +788,21 @@ class TReplayRequestGeneratorFs final auto newFullName = Spec.GetReplayRoot() + PathByNode(newparentnodeid) + r.GetNodeInfo().GetNewNodeName(); + + DUMP("prerename", NFs::Exists(fullName), NFs::Exists(newFullName)); + const auto renameres = NFs::Rename(fullName, newFullName); + DUMP(LastSystemError(), LastSystemErrorText(LastSystemError())); + + DUMP("pstrename", NFs::Exists(fullName), NFs::Exists(newFullName)); + STORAGE_DEBUG( - "rename " << fullName << " => " << newFullName << " : " - << renameres); + "rename " << fullName << " => " << newFullName << " : " << renameres + << (renameres + ? (TStringBuilder{} + << " errno=" << LastSystemError() << " err=" + << LastSystemErrorText(LastSystemError())) + : TStringBuilder{})); return MakeFuture( TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); } @@ -829,7 +876,7 @@ class TReplayRequestGeneratorFs final OpenHandles.erase(handleid); HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); STORAGE_DEBUG( - "Close " << handleid << " orig=" << r.GetNodeInfo().GetHandle() + "Close " << handleid << " <- " << r.GetNodeInfo().GetHandle() << " pos=" << pos << " len=" << len << " open map size=" << OpenHandles.size() << " map size=" << HandlesLogToActual.size()); diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp index a4238cc00c..151076c6ec 100644 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ b/cloud/filestore/tools/testing/replay/lib/test.cpp @@ -621,11 +621,14 @@ class TLoadTest final auto code = request->Error.GetCode(); if (FAILED(code)) { STORAGE_WARN( - "%s failed request: %s", + "%s failed request %d: %s", MakeTestTag().c_str(), + request->Action, FormatError(request->Error).c_str()); - // TestStats.Success = false; + if (RequestGenerator->FailOnError()) { + TestStats.Success = false; + } } if (request->Stop) { diff --git a/cloud/filestore/tools/testing/replay/protos/replay.proto b/cloud/filestore/tools/testing/replay/protos/replay.proto index c174cd5eae..025e3f6392 100644 --- a/cloud/filestore/tools/testing/replay/protos/replay.proto +++ b/cloud/filestore/tools/testing/replay/protos/replay.proto @@ -40,6 +40,7 @@ message TReplaySpec double TimeScale = 12; bool WriteRandom = 13; bool WriteEmpty = 14; + bool CreateOnRead = 15; } message TMigrationSpec From a297662467f59922389609b206c43b78e6a0b878 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 25 Sep 2024 16:15:36 +0000 Subject: [PATCH 09/44] merge to loadtest --- .../tools/testing/loadtest/lib/request.h | 31 +- .../lib/request_replay_fs.cpp | 16 +- .../loadtest/lib/request_replay_grpc.cpp | 1136 ++++++++++++++++ .../tools/testing/loadtest/lib/test.cpp | 34 +- .../tools/testing/loadtest/lib/ya.make | 13 +- .../testing/loadtest/protos/loadtest.proto | 16 + .../tools/testing/replay/bin/app.cpp | 187 --- .../filestore/tools/testing/replay/bin/app.h | 14 - .../tools/testing/replay/bin/bootstrap.cpp | 202 --- .../tools/testing/replay/bin/bootstrap.h | 72 - .../tools/testing/replay/bin/main.cpp | 43 - .../tools/testing/replay/bin/options.cpp | 90 -- .../tools/testing/replay/bin/options.h | 40 - .../tools/testing/replay/bin/private.h | 14 - .../tools/testing/replay/bin/ya.make | 29 - .../tools/testing/replay/lib/client.cpp | 1 - .../tools/testing/replay/lib/client.h | 21 - .../tools/testing/replay/lib/context.h | 18 - .../tools/testing/replay/lib/executor.cpp | 60 - .../tools/testing/replay/lib/executor.h | 40 - .../tools/testing/replay/lib/public.h | 20 - .../tools/testing/replay/lib/request.h | 77 -- .../tools/testing/replay/lib/test.cpp | 1156 ----------------- .../filestore/tools/testing/replay/lib/test.h | 38 - .../tools/testing/replay/lib/ya.make | 36 - .../tools/testing/replay/protos/replay.proto | 126 -- .../tools/testing/replay/protos/ya.make | 13 - cloud/filestore/tools/testing/replay/ya.make | 5 - 28 files changed, 1228 insertions(+), 2320 deletions(-) rename cloud/filestore/tools/testing/{replay => loadtest}/lib/request_replay_fs.cpp (99%) create mode 100644 cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp delete mode 100644 cloud/filestore/tools/testing/replay/bin/app.cpp delete mode 100644 cloud/filestore/tools/testing/replay/bin/app.h delete mode 100644 cloud/filestore/tools/testing/replay/bin/bootstrap.cpp delete mode 100644 cloud/filestore/tools/testing/replay/bin/bootstrap.h delete mode 100644 cloud/filestore/tools/testing/replay/bin/main.cpp delete mode 100644 cloud/filestore/tools/testing/replay/bin/options.cpp delete mode 100644 cloud/filestore/tools/testing/replay/bin/options.h delete mode 100644 cloud/filestore/tools/testing/replay/bin/private.h delete mode 100644 cloud/filestore/tools/testing/replay/bin/ya.make delete mode 100644 cloud/filestore/tools/testing/replay/lib/client.cpp delete mode 100644 cloud/filestore/tools/testing/replay/lib/client.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/context.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/executor.cpp delete mode 100644 cloud/filestore/tools/testing/replay/lib/executor.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/public.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/request.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/test.cpp delete mode 100644 cloud/filestore/tools/testing/replay/lib/test.h delete mode 100644 cloud/filestore/tools/testing/replay/lib/ya.make delete mode 100644 cloud/filestore/tools/testing/replay/protos/replay.proto delete mode 100644 cloud/filestore/tools/testing/replay/protos/ya.make delete mode 100644 cloud/filestore/tools/testing/replay/ya.make diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 396e1f8eb0..74abd8abdc 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -21,9 +21,10 @@ namespace NCloud::NFileStore::NLoadTest { struct TCompletedRequest { - NProto::EAction Action; + NProto::EAction Action{}; TDuration Elapsed; NProto::TError Error; + bool Stop{}; TCompletedRequest() = default; @@ -32,6 +33,10 @@ struct TCompletedRequest , Elapsed(TInstant::Now() - start) , Error(std::move(error)) {} + + TCompletedRequest(bool stop) noexcept + : Stop{stop} + {} }; //////////////////////////////////////////////////////////////////////////////// @@ -43,6 +48,16 @@ struct IRequestGenerator virtual bool HasNextRequest() = 0; virtual TInstant NextRequestAt() = 0; virtual NThreading::TFuture ExecuteNextRequest() = 0; + + virtual bool InstantProcessQueue() + { + return false; + } + + virtual bool FailOnError() + { + return true; + } }; //////////////////////////////////////////////////////////////////////////////// @@ -61,4 +76,18 @@ IRequestGeneratorPtr CreateDataRequestGenerator( TString filesystemId, NProto::THeaders headers); +IRequestGeneratorPtr CreateReplayRequestGeneratorFs( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers); + +IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers); + } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp similarity index 99% rename from cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp rename to cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 015c29448b..dd19fad252 100644 --- a/cloud/filestore/tools/testing/replay/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -1,5 +1,4 @@ /* - TODO: create file/dir modes create handle modes (now rw) @@ -11,7 +10,6 @@ read/write with multiranges (now only first processed) #include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" #include "cloud/filestore/tools/analytics/libs/event-log/dump.h" -#include "dump.h" #include "library/cpp/aio/aio.h" #include "library/cpp/eventlog/iterator.h" #include "library/cpp/testing/unittest/registar.h" @@ -39,7 +37,7 @@ read/write with multiranges (now only first processed) #include -namespace NCloud::NFileStore::NReplay { +namespace NCloud::NFileStore::NLoadTest { using namespace NThreading; using namespace NCloud::NFileStore::NClient; @@ -231,6 +229,11 @@ class TReplayRequestGeneratorFs final return !!EventPtr; } + TInstant NextRequestAt() override + { + return TInstant::Max(); + } + NThreading::TFuture ExecuteNextRequest() override { if (!HasNextRequest()) { @@ -789,12 +792,7 @@ class TReplayRequestGeneratorFs final auto newFullName = Spec.GetReplayRoot() + PathByNode(newparentnodeid) + r.GetNodeInfo().GetNewNodeName(); - DUMP("prerename", NFs::Exists(fullName), NFs::Exists(newFullName)); - const auto renameres = NFs::Rename(fullName, newFullName); - DUMP(LastSystemError(), LastSystemErrorText(LastSystemError())); - - DUMP("pstrename", NFs::Exists(fullName), NFs::Exists(newFullName)); STORAGE_DEBUG( "rename " << fullName << " => " << newFullName << " : " << renameres @@ -1135,4 +1133,4 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorFs( std::move(headers)); } -} // namespace NCloud::NFileStore::NReplay +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp new file mode 100644 index 0000000000..677fa9e8e8 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -0,0 +1,1136 @@ +/* + +TODO: +create file/dir modes +create handle modes (now rw) +compare log and actual result ( S_OK E_FS_NOENT ...) + +*/ + +#include "request.h" + +#include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" +#include "cloud/filestore/tools/analytics/libs/event-log/dump.h" +#include "library/cpp/eventlog/iterator.h" +#include "library/cpp/testing/unittest/registar.h" +#include "util/folder/dirut.h" +#include "util/folder/path.h" +#include "util/system/fs.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NLoadTest { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +namespace { + +//////////////////////////////////////////////////////////////////////////////// + +class TReplayRequestGeneratorGRPC final + : public IRequestGenerator + , public std::enable_shared_from_this +{ +private: + const NProto::TReplaySpec Spec; + TString FileSystemId; + const NProto::THeaders Headers; + + TLog Log; + + ISessionPtr Session; + + TVector> Actions; + + TMutex StateLock; + + using THandleLog = ui64; + using THandleLocal = ui64; + using TNodeLog = ui64; + using TNodeLocal = ui64; + + ui64 InitialFileSize = 0; + + std::atomic LastRequestId = 0; + + THolder EventlogIterator; + TConstEventPtr EventPtr; + int EventMessageNumber = 0; + NProto::TProfileLogRecord* EventLogMessagePtr{}; + + static constexpr ui32 LockLength = 4096; + + const ui64 OwnerId = RandomNumber(100500u); + + struct TNode + { + TString Name; + NProto::TNodeAttr Attrs; + + TNodeLog ParentLog = 0; + }; + + struct THandle + { + TString Path; + ui64 Handle = 0; + }; + + THashMap Handles; + THashSet Locks; + THashSet StagedLocks; + + THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; + THashMap NodePath{{RootNodeId, "/"}}; + THashMap KnownLogNodes; + + THashMap HandlesLogToActual; + // THashMap OpenHandles; + + ui64 TimestampMcs{}; + TInstant Started; + +public: + TReplayRequestGeneratorGRPC( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) + { + Log = logging->CreateLog(Headers.GetClientId()); + + NEventLog::TOptions options; + options.FileName = Spec.GetFileName(); + options.SetForceStrongOrdering(true); // need this? + EventlogIterator = CreateIterator(options); + } + + ~TReplayRequestGeneratorGRPC() + {} + + bool InstantProcessQueue() override + { + return true; + } + bool FailOnError() override + { + return false; + } + + TNodeLocal NodeIdMapped(const TNodeLog id) + { + if (const auto it = NodesLogToLocal.find(id); + it != NodesLogToLocal.end()) + { + return it->second; + } + + STORAGE_DEBUG( + "node not found " << id << " map size=" << NodesLogToLocal.size()); + return 0; + } + + THandleLocal HandleIdMapped(const THandleLog id) + { + if (const auto it = HandlesLogToActual.find(id); + it != HandlesLogToActual.end()) + { + return it->second; + } + STORAGE_DEBUG( + "handle not found " << id + << " map size=" << HandlesLogToActual.size()); + return 0; + } + + void Advance() + { + EventPtr = EventlogIterator->Next(); + if (!EventPtr) { + return; + } + + EventLogMessagePtr = const_cast( + dynamic_cast( + EventPtr->GetProto())); + if (!EventLogMessagePtr) { + return; + } + + if (FileSystemId.empty()) { + FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; + } + + EventMessageNumber = EventLogMessagePtr->GetRequests().size(); + } + + bool HasNextRequest() override + { + if (!EventPtr) { + Advance(); + } + return !!EventPtr; + } + + TInstant NextRequestAt() override + { + return TInstant::Max(); + } + + NThreading::TFuture ExecuteNextRequest() override + { + if (!HasNextRequest()) { + return MakeFuture(TCompletedRequest{}); + } + for (; EventPtr; Advance()) { + if (!EventLogMessagePtr) { + continue; + } + + STORAGE_DEBUG("Processing %d", EventMessageNumber); + for (; EventMessageNumber > 0;) { + auto request = + EventLogMessagePtr->GetRequests()[--EventMessageNumber]; + STORAGE_DEBUG("message json=" << request.AsJSON()); + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = + TDuration::MicroSeconds(timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); + + Sleep(slp); + } + Started = current; + + STORAGE_DEBUG( + "Processing message " + << EventMessageNumber + << " typename=" << request.GetTypeName() + << " type=" << request.GetRequestType() << " " + << RequestName(request.GetRequestType())); + + { + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest::ReadData: + return DoReadData(request); + case EFileStoreRequest::WriteData: + return DoWrite(request); + case EFileStoreRequest::CreateNode: + return DoCreateNode(request); + case EFileStoreRequest::RenameNode: + return DoRenameNode(request); + case EFileStoreRequest::UnlinkNode: + return DoUnlinkNode(request); + case EFileStoreRequest::CreateHandle: + return DoCreateHandle(request); + case EFileStoreRequest::DestroyHandle: + return DoDestroyHandle(request); + case EFileStoreRequest::GetNodeAttr: + return DoGetNodeAttr(request); + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(); + + case EFileStoreRequest::AccessNode: + return DoAccessNode(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + continue; + default: + STORAGE_INFO( + "Uninmplemented action=" + << action << " " + << RequestName(request.GetRequestType())); + continue; + } + } + } + } + STORAGE_INFO( + "Log finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); + + return MakeFuture(TCompletedRequest(true)); + } + +private: + TFuture DoAccessNode( + [[maybe_unused]] const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& r) + { + // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} + + auto started = TInstant::Now(); + + if (Spec.GetNoRead()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACCESS_NODE, + started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + } + + TFuture DoCreateHandle( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} + // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} + // nfs CreateHandle 0.004161s S_OK {parent_node_id=65, + // node_name=ini, flags=14, mode=436, node_id=66, + // handle=11024287581389312, size=0} + + TGuard guard(StateLock); + auto started = TInstant::Now(); + + auto request = CreateRequest(); + auto name = r.GetNodeInfo().GetNodeName(); + + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + request->SetMode(r.GetNodeInfo().GetMode()); + + auto self = weak_from_this(); + return Session->CreateHandle(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr + ->HandleCreateHandle(future, name, started, r); + } + + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + MakeError(E_FAIL, "cancelled")}); + }); + } + + TFuture HandleCreateHandle( + const TFuture& future, + const TString& name, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + auto handle = response.GetHandle(); + with_lock (StateLock) { + HandlesLogToActual[r.GetNodeInfo().GetHandle()] = handle; + } + + NThreading::TFuture setAttr; + if (InitialFileSize) { + static const int flags = + ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetNodeId(response.GetNodeAttr().GetId()); + request->SetFlags(flags); + request->MutableUpdate()->SetSize(InitialFileSize); + + setAttr = Session->SetNodeAttr( + CreateCallContext(), + std::move(request)); + } else { + setAttr = + NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); + } + + return setAttr.Apply( + [=]([[maybe_unused]] const TFuture< + NProto::TSetNodeAttrResponse>& f) + { + return MakeFuture(TCompletedRequest{}); + // return HandleResizeAfterCreateHandle(f, name, started); + }); + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "create handle for %s has failed: %s", + name.Quote().c_str(), + FormatError(error).c_str()); + + return NThreading::MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + started, + error}); + } + } + + static constexpr ui32 BlockSize = 4_KB; + static std::shared_ptr Acalloc(ui64 dataSize) + { + std::shared_ptr buffer = { + static_cast(aligned_alloc(BlockSize, dataSize)), + [](auto* p) + { + free(p); + }}; + memset(buffer.get(), 0, dataSize); + + return buffer; + } + TFuture DoReadData( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + const auto started = TInstant::Now(); + if (Spec.GetNoRead()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + auto request = CreateRequest(); + const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + request->SetHandle(handle); + request->SetOffset(r.GetRanges().cbegin()->GetOffset()); + request->SetLength(r.GetRanges().cbegin()->GetBytes()); + + auto self = weak_from_this(); + return Session->ReadData(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + return TCompletedRequest{ + NProto::ACTION_READ, + started, + future.GetValue().GetError()}; + /* + if (auto ptr = self.lock()) { + return ptr->HandleRead( + future, + handleInfo, + started, + byteOffset); + } + */ + return TCompletedRequest{ + NProto::ACTION_READ, + started, + MakeError(E_FAIL, "cancelled")}; + }); + } + + TCompletedRequest HandleRead( + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + return {NProto::ACTION_READ, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "read for %s has failed: ", + // handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_READ, started, error}; + } + } + + TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) + { + TStringBuilder ret; + ret << "[\n" << start; + while (ret.size() < bytes) { + ret << " . " << ret.size() << " : " << offset + ret.size(); + } + return ret.substr(0, bytes); + } + + TFuture DoWrite( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} + + const auto started = TInstant::Now(); + if (Spec.GetNoWrite()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_WRITE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + TGuard guard(StateLock); + + auto request = CreateRequest(); + const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); + + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + request->SetHandle(handle); + + request->SetOffset(r.GetRanges(0).GetOffset()); + auto bytes = r.GetRanges(0).GetBytes(); + auto buffer = NUnitTest::RandomString(bytes); + + *request->MutableBuffer() = std::move(buffer); + + auto self = weak_from_this(); + return Session->WriteData(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleWrite(future, started); + } + + return TCompletedRequest{ + NProto::ACTION_WRITE, + started, + MakeError(E_FAIL, "cancelled")}; + }); + } + + TCompletedRequest HandleWrite( + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_WRITE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + /* + STORAGE_ERROR( + "write on %s has failed: %s", + handleInfo.Name.Quote().c_str(), + FormatError(error).c_str()); + */ + return {NProto::ACTION_WRITE, started, error}; + } + } + + TString PathByNode(TNodeLocal nodeid) + { + if (const auto& it = NodePath.find(nodeid); it != NodePath.end()) { + return it->second; + } + return {}; + } + + static TNodeLocal MakeDirectoryRecursive(const TString& name) + { + NFs::MakeDirectoryRecursive(name); + const auto inode = TFileStat{name}.INode; + return inode; + } + + TFuture DoCreateNode( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} + // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, + // new_node_name=home, mode=509, node_id=12526, size=0} + + const auto started = TInstant::Now(); + if (Spec.GetNoWrite()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + auto request = CreateRequest(); + + const auto parentNode = + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + if (!parentNode) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(parentNode); + auto name = r.GetNodeInfo().GetNewNodeName(); + request->SetName(r.GetNodeInfo().GetNewNodeName()); + + // request->SetGid(); + // request->SetUid(); + + switch (r.GetNodeInfo().GetType()) { + case NProto::E_REGULAR_NODE: + request->MutableFile()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_DIRECTORY_NODE: + request->MutableDirectory()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_LINK_NODE: + // TODO: + // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); + // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); + return MakeFuture(TCompletedRequest{}); + break; + case NProto::E_SYMLINK_NODE: + // TODO: + // request->MutableSymlink()->SetTargetPath(); + break; + case NProto::E_SOCK_NODE: + request->MutableSocket()->SetMode(r.GetNodeInfo().GetMode()); + break; + case NProto::E_INVALID_NODE: + return MakeFuture( + TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); + // Do not create files with invalid + // type - too hard to delete them + break; + } + Cerr << "createnoderec" << *request << "\n"; + auto self = weak_from_this(); + return Session->CreateNode(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleCreateNode(future, name, started, r); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleCreateNode( + const TFuture& future, + const TString& name, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + TGuard guard(StateLock); + + try { + auto response = future.GetValue(); + CheckResponse(response); + // Nodes[name] = TNode{name, response.GetNode()}; + if (response.GetNode().GetId()) { + NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = + response.GetNode().GetId(); + } + + return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "create node %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_CREATE_NODE, started, error}; + } + } + + TFuture DoRenameNode( + // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} + // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, + // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} + // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); + + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + auto started = TInstant::Now(); + if (Spec.GetNoRead()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + auto request = CreateRequest(); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError( + E_FAIL, + TStringBuilder{} << "Log node " + << r.GetNodeInfo().GetParentNodeId() + << "not found in mappiong")}); + } + request->SetNodeId(node); + request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetNewParentId( + NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId())); + request->SetNewName(r.GetNodeInfo().GetNewNodeName()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + + auto self = weak_from_this(); + return Session->RenameNode(CreateCallContext(), std::move(request)) + .Apply( + [= + + ](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleRenameNode(future, started, r); + } + + return TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleRenameNode( + const TFuture& future, + TInstant started, + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + TGuard guard(StateLock); + try { + auto response = future.GetValue(); + CheckResponse(response); + return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "rename node %s has failed: %s", + r.GetNodeInfo().GetNodeName().c_str(), + FormatError(error).c_str()); + return {NProto::ACTION_RENAME_NODE, started, error}; + } + } + + TFuture DoUnlinkNode( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + // UnlinkNode 0.002605s S_OK {parent_node_id=3, + // node_name=tfrgYZ1} + + auto started = TInstant::Now(); + if (Spec.GetNoWrite()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + auto name = r.GetNodeInfo().GetNodeName(); + auto request = CreateRequest(); + request->SetName(name); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + auto self = weak_from_this(); + return Session->UnlinkNode(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleUnlinkNode(future, name, started); + } + + return TCompletedRequest{ + NProto::ACTION_REMOVE_NODE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleUnlinkNode( + const TFuture& future, + const TString& name, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "unlink for %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_REMOVE_NODE, started, error}; + } + } + TFuture DoDestroyHandle( + NCloud::NFileStore::NProto::TProfileLogRequestInfo r) + { + // DestroyHandle 0.002475s S_OK {node_id=10, + // handle=61465562388172112} + TGuard guard(StateLock); + auto started = TInstant::Now(); + + auto name = r.GetNodeInfo().GetNodeName(); + + auto request = CreateRequest(); + + const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + if (!handle) { + return MakeFuture(TCompletedRequest{}); + } + + HandlesLogToActual.erase(handle); + + request->SetHandle(handle); + + auto self = weak_from_this(); + return Session->DestroyHandle(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleDestroyHandle(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_DESTROY_HANDLE, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TFuture DoGetNodeAttr( + NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + { + const auto started = TInstant::Now(); + if (Spec.GetNoRead()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_PRECONDITION_FAILED, "disabled")}); + } + + TGuard guard(StateLock); + + // TODO: by parent + name // + // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} + // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, + // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} + + auto request = CreateRequest(); + const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + if (!node) { + return MakeFuture(TCompletedRequest{}); + } + request->SetNodeId(node); + auto name = r.GetNodeInfo().GetNodeName(); + request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetFlags(r.GetNodeInfo().GetFlags()); + auto self = weak_from_this(); + STORAGE_DEBUG("GetNodeAttr client started"); + return Session->GetNodeAttr(CreateCallContext(), std::move(request)) + .Apply( + [=, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleGetNodeAttr(name, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleGetNodeAttr( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + STORAGE_DEBUG("GetNodeAttr client completed"); + CheckResponse(response); + TGuard guard(StateLock); + return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "get node attr %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + } + } + + TCompletedRequest HandleDestroyHandle( + const TString& name, + const TFuture& future, + TInstant started) + { + try { + auto response = future.GetValue(); + CheckResponse(response); + + return {NProto::ACTION_DESTROY_HANDLE, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "destroy handle %s has failed: %s", + name.c_str(), + FormatError(error).c_str()); + + return {NProto::ACTION_DESTROY_HANDLE, started, error}; + } + } + + TFuture DoAcquireLock() + { + TGuard guard(StateLock); + if (Handles.empty()) { + // return DoCreateHandle(); + } + + auto started = TInstant::Now(); + auto it = Handles.begin(); + while (it != Handles.end() && + (Locks.contains(it->first) || StagedLocks.contains(it->first))) + { + ++it; + } + + if (it == Handles.end()) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}); + } + + auto handle = it->first; + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto self = weak_from_this(); + return Session->AcquireLock(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleAcquireLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleAcquireLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it == StagedLocks.end()) { + // nothing todo, file was removed + Y_ABORT_UNLESS(!Locks.contains(handle)); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } + + StagedLocks.erase(it); + + try { + const auto& response = future.GetValue(); + CheckResponse(response); + + Locks.insert(handle); + return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "acquire lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_ACQUIRE_LOCK, started, error}; + } + } + + TFuture DoReleaseLock() + { + TGuard guard(StateLock); + if (Locks.empty()) { + return DoAcquireLock(); + } + + auto it = Locks.begin(); + auto handle = *it; + + Y_ABORT_UNLESS(StagedLocks.insert(handle).second); + Locks.erase(it); + + auto request = CreateRequest(); + request->SetHandle(handle); + request->SetOwner(OwnerId); + request->SetLength(LockLength); + + auto started = TInstant::Now(); + auto self = weak_from_this(); + return Session->ReleaseLock(CreateCallContext(), std::move(request)) + .Apply( + [=](const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleReleaseLock(handle, future, started); + } + + return TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + started, + MakeError(E_CANCELLED, "cancelled")}; + }); + } + + TCompletedRequest HandleReleaseLock( + ui64 handle, + const TFuture& future, + TInstant started) + { + TGuard guard(StateLock); + + auto it = StagedLocks.find(handle); + if (it != StagedLocks.end()) { + StagedLocks.erase(it); + } + + try { + CheckResponse(future.GetValue()); + return {NProto::ACTION_RELEASE_LOCK, started, {}}; + } catch (const TServiceError& e) { + auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); + STORAGE_ERROR( + "release lock on %lu has failed: %s", + handle, + FormatError(error).c_str()); + + return {NProto::ACTION_RELEASE_LOCK, started, error}; + } + } + + template + std::shared_ptr CreateRequest() + { + auto request = std::make_shared(); + request->SetFileSystemId(FileSystemId); + request->MutableHeaders()->CopyFrom(Headers); + + return request; + } + + template + void CheckResponse(const T& response) + { + if (HasError(response)) { + throw TServiceError(response.GetError()); + } + } + + TIntrusivePtr CreateCallContext() + { + return MakeIntrusive( + LastRequestId.fetch_add(1, std::memory_order_relaxed)); + } +}; + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + +IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) +{ + return std::make_shared( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)); +} + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index f9704a16be..b930702695 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -532,6 +532,23 @@ class TLoadTest final FileSystemId, headers); break; + case NProto::TLoadTest::kReplayFsSpec: + RequestGenerator = CreateReplayRequestGeneratorFs( + Config.GetReplayFsSpec(), + Logging, + Session, + FileSystemId, + headers); + break; + + case NProto::TLoadTest::kReplayGrpcSpec: + RequestGenerator = CreateReplayRequestGeneratorGRPC( + Config.GetReplayGrpcSpec(), + Logging, + Session, + FileSystemId, + headers); + break; default: ythrow yexception() << MakeTestTag() @@ -576,13 +593,16 @@ class TLoadTest final ++CurrentIoDepth; auto self = weak_from_this(); - RequestGenerator->ExecuteNextRequest().Apply( - [=] (const TFuture& future) { + const auto future = RequestGenerator->ExecuteNextRequest().Apply( if (auto ptr = self.lock()) { ptr->SignalCompletion(future.GetValue()); } }); - + if (RequestGenerator->InstantProcessQueue()) { + if (future.HasValue()) { + ProcessCompletedRequests(); + } + } return true; } @@ -608,12 +628,18 @@ class TLoadTest final MakeTestTag().c_str(), FormatError(request->Error).c_str()); - TestStats.Success = false; + if (RequestGenerator->FailOnError()) { + TestStats.Success = false; + } } auto& stats = TestStats.ActionStats[request->Action]; ++stats.Requests; stats.Hist.RecordValue(request->Elapsed); + + if (request->Stop) { + TestStats.Success = false; + } } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/ya.make b/cloud/filestore/tools/testing/loadtest/lib/ya.make index 8b8e7e5929..5ec90fbf77 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/ya.make +++ b/cloud/filestore/tools/testing/loadtest/lib/ya.make @@ -6,22 +6,27 @@ SRCS( executor.cpp request_data.cpp request_index.cpp + request_replay_fs.cpp + request_replay_grpc.cpp test.cpp ) PEERDIR( cloud/filestore/config - cloud/filestore/public/api/protos - cloud/filestore/libs/client + cloud/filestore/libs/diagnostics/events cloud/filestore/libs/service + cloud/filestore/libs/service_local + cloud/filestore/public/api/protos + cloud/filestore/tools/analytics/libs/event-log + cloud/filestore/tools/testing/loadtest/protos cloud/storage/core/libs/common cloud/storage/core/libs/diagnostics - cloud/filestore/tools/testing/loadtest/protos - + library/cpp/aio library/cpp/deprecated/atomic + library/cpp/testing/unittest ) END() diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index 353f3d88f8..ee2f953677 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -18,6 +18,8 @@ enum EAction ACTION_ACQUIRE_LOCK = 5; ACTION_RELEASE_LOCK = 6; ACTION_GET_NODE_ATTR = 7; + ACTION_ACCESS_NODE = 8; + ACTION_LIST_NODES = 9; // DATA ACTION_WRITE = 101; @@ -54,6 +56,18 @@ message TDataLoadSpec repeated TAction Actions = 8; } +message TReplaySpec +{ + string FileName = 1; + string ReplayRoot = 8; // not used in grpc + bool NoRead = 9; + bool NoWrite = 10; + bool Prepare = 11; // Only write files needed for reading + double TimeScale = 12; + bool WriteRandom = 13; + bool WriteEmpty = 14; + bool CreateOnRead = 15; +} message TMigrationSpec { @@ -78,6 +92,8 @@ message TLoadTest { TIndexLoadSpec IndexLoadSpec = 4; TDataLoadSpec DataLoadSpec = 5; + TReplaySpec ReplayFsSpec = 17; + TReplaySpec ReplayGrpcSpec = 18; } uint32 IODepth = 10; diff --git a/cloud/filestore/tools/testing/replay/bin/app.cpp b/cloud/filestore/tools/testing/replay/bin/app.cpp deleted file mode 100644 index 707a39132d..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/app.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "app.h" - -#include "bootstrap.h" -#include "options.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include - -namespace NCloud::NFileStore::NReplay { - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -class TApp -{ -private: - TLog Log; - - TAppContext Context; - - TMutex Lock; - TMap Tests; - -public: - static TApp* GetInstance() - { - return Singleton(); - } - - int Run(TBootstrap& bootstrap) - { - NCloud::SetCurrentThreadName("main"); - Log = bootstrap.GetLogging()->CreateLog("MAIN"); - - try { - NProto::TTestGraph graph; - - auto options = bootstrap.GetOptions(); - if (options->TestsConfig) { - ParseFromTextFormat(options->TestsConfig, graph); - } - - TVector> tests; - for (ui32 i = 0; i < graph.TestsSize(); ++i) { - const auto& entry = graph.GetTests(i); - - switch (entry.GetTestCase()) { - case NProto::TTestGraph::TTest::kLoadTest: { - auto name = entry.GetLoadTest().GetName(); - - auto test = CreateLoadTest( - Context, - entry.GetLoadTest(), - bootstrap.GetTimer(), - bootstrap.GetScheduler(), - bootstrap.GetLogging(), - bootstrap.GetClientFactory()); - - tests.push_back([=, this]() - { RunLoadTest(i, name, test); }); - - break; - } - default: - Y_ABORT( - "unexpected test case: %s", - entry.ShortDebugString().c_str()); - } - } - - TExecutor executor(Context, std::move(tests)); - executor.Run(); - - for (const auto& pair: Tests) { - bootstrap.GetOptions()->GetOutputStream() - << NProtobufJson::Proto2Json( - pair.second, - {.FormatOutput = true}) - << Endl; - } - - } catch (...) { - STORAGE_ERROR("app has failed: " << CurrentExceptionMessage()); - return 1; - } - - return AtomicGet(Context.ExitCode); - } - - void RunLoadTest(ui32 i, const TString& name, const ITestPtr& test) - { - try { - const auto future = test->Run(); - if (!future.Initialized()) { - return; - } - const auto& stats = future.GetValueSync(); - if (!stats.GetSuccess()) { - ythrow yexception() << "not successful"; - } - - with_lock (Lock) { - Tests[i] = stats; - } - - STORAGE_INFO("%s has finished", MakeTestTag(name).c_str()); - - } catch (...) { - STORAGE_INFO( - "%s test has failed: %s", - MakeTestTag(name).c_str(), - CurrentExceptionMessage().c_str()); - - Stop(1); - } - } - - void Stop(int exitCode) - { - AtomicSet(Context.ExitCode, exitCode); - AtomicSet(Context.ShouldStop, 1); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -void ProcessSignal(int signum) -{ - if (signum == SIGINT || signum == SIGTERM) { - AppStop(0); - } -} - -void ProcessAsyncSignal(int signum) -{ - if (signum == SIGHUP) { - TLogBackend::ReopenAllBackends(); - } -} - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -void ConfigureSignals() -{ - std::set_new_handler(abort); - - // make sure that errors can be seen by everybody :) - setvbuf(stdout, nullptr, _IONBF, 0); - setvbuf(stderr, nullptr, _IONBF, 0); - - // mask signals - signal(SIGPIPE, SIG_IGN); - signal(SIGINT, ProcessSignal); - signal(SIGTERM, ProcessSignal); - - SetAsyncSignalHandler(SIGHUP, ProcessAsyncSignal); -} - -int AppMain(TBootstrap& bootstrap) -{ - return TApp::GetInstance()->Run(bootstrap); -} - -void AppStop(int exitCode) -{ - TApp::GetInstance()->Stop(exitCode); -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/app.h b/cloud/filestore/tools/testing/replay/bin/app.h deleted file mode 100644 index bd58a22d72..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/app.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "private.h" - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -void ConfigureSignals(); - -int AppMain(TBootstrap& bootstrap); -void AppStop(int exitCode); - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp b/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp deleted file mode 100644 index 415cb3cfa1..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/bootstrap.cpp +++ /dev/null @@ -1,202 +0,0 @@ -#include "bootstrap.h" - -#include "options.h" - -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -#include -#include - -#include -#include - -namespace NCloud::NFileStore::NReplay { - -using namespace NMonitoring; - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -class TClientFactory final - : public IClientFactory -{ -private: - const ILoggingServicePtr Logging; - const ITimerPtr Timer; - const ISchedulerPtr Scheduler; - const NClient::TClientConfigPtr ClientConfig; - - TVector Clients; - -public: - TClientFactory( - ILoggingServicePtr logging, - ITimerPtr timer, - ISchedulerPtr scheduler, - NClient::TClientConfigPtr clientConfig) - : Logging(std::move(logging)) - , Timer(std::move(timer)) - , Scheduler(std::move(scheduler)) - , ClientConfig(std::move(clientConfig)) - {} - - void Start() override - { - for (auto& client: Clients) { - client->Start(); - } - } - - void Stop() override - { - for (auto& client: Clients) { - client->Stop(); - } - } - - IFileStoreServicePtr CreateClient() override - { - auto client = NClient::CreateFileStoreClient( - ClientConfig, - Logging); - - client = NClient::CreateDurableClient( - Logging, - Timer, - Scheduler, - CreateRetryPolicy(ClientConfig), - client); - - Clients.push_back(client); - return client; - } -}; - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -TBootstrap::TBootstrap(TOptionsPtr options) - : Options(std::move(options)) -{} - -TBootstrap::~TBootstrap() -{} - -void TBootstrap::Init() -{ - InitDbgConfig(); - InitClientConfig(); - - Timer = CreateWallClockTimer(); - Scheduler = CreateScheduler(); - - ClientFactory = std::make_shared( - Logging, - Timer, - Scheduler, - ClientConfig); -} - -void TBootstrap::Start() -{ - if (Logging) { - Logging->Start(); - } - - if (Monitoring) { - Monitoring->Start(); - } - - if (Scheduler) { - Scheduler->Start(); - } - - if (ClientFactory) { - ClientFactory->Start(); - } -} - -void TBootstrap::Stop() -{ - if (ClientFactory) { - ClientFactory->Stop(); - } - - if (Scheduler) { - Scheduler->Stop(); - } - - if (Monitoring) { - Monitoring->Stop(); - } - - if (Logging) { - Logging->Stop(); - } -} - -//////////////////////////////////////////////////////////////////////////////// - -void TBootstrap::InitDbgConfig() -{ - TLogSettings logSettings; - - if (Options->VerboseLevel) { - auto level = GetLogLevel(Options->VerboseLevel); - Y_ENSURE(level, "unknown log level: " << Options->VerboseLevel.Quote()); - logSettings.FiltrationLevel = *level; - } - - Logging = CreateLoggingService("console", logSettings); - Log = Logging->CreateLog("BOOT"); - - if (Options->MonitoringPort) { - Monitoring = CreateMonitoringService( - Options->MonitoringPort, - Options->MonitoringAddress, - Options->MonitoringThreads); - } else { - Monitoring = CreateMonitoringServiceStub(); - } -} - -void TBootstrap::InitClientConfig() -{ - NProto::TClientConfig clientConfig; - - if (Options->ConfigFile) { - ParseFromTextFormat(Options->ConfigFile, clientConfig); - } - if (Options->Host) { - clientConfig.SetHost(Options->Host); - } - if (Options->InsecurePort) { - clientConfig.SetPort(Options->InsecurePort); - } - if (Options->SecurePort) { - clientConfig.SetSecurePort(Options->SecurePort); - } - - ClientConfig = std::make_shared(clientConfig); - - TStringStream ss; - ClientConfig->Dump(ss); - - STORAGE_INFO("client config:\n" << ss.Str()); -} - -} // namespace NCloud::NFileStore::NServer diff --git a/cloud/filestore/tools/testing/replay/bin/bootstrap.h b/cloud/filestore/tools/testing/replay/bin/bootstrap.h deleted file mode 100644 index e83e4de116..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/bootstrap.h +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include "private.h" - -#include - -#include -#include - -#include -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -class TBootstrap -{ -private: - TOptionsPtr Options; - - ILoggingServicePtr Logging; - TLog Log; - - IMonitoringServicePtr Monitoring; - ITimerPtr Timer; - ISchedulerPtr Scheduler; - - NClient::TClientConfigPtr ClientConfig; - IClientFactoryPtr ClientFactory; - -public: - TBootstrap(TOptionsPtr options); - ~TBootstrap(); - - void Init(); - void Start(); - void Stop(); - - TOptionsPtr GetOptions() - { - return Options; - } - - ITimerPtr GetTimer() - { - return Timer; - } - - ISchedulerPtr GetScheduler() - { - return Scheduler; - } - - ILoggingServicePtr GetLogging() - { - return Logging; - } - - IClientFactoryPtr GetClientFactory() - { - return ClientFactory; - } - -private: - void InitDbgConfig(); - void InitClientConfig(); -}; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/main.cpp b/cloud/filestore/tools/testing/replay/bin/main.cpp deleted file mode 100644 index fa4f75b44d..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/main.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "app.h" -#include "bootstrap.h" -#include "options.h" - -#include - -//////////////////////////////////////////////////////////////////////////////// - -int main(int argc, char** argv) -{ - using namespace NCloud::NFileStore::NReplay; - - ConfigureSignals(); - - auto options = std::make_shared(); - try { - options->Parse(argc, argv); - } catch (...) { - Cerr << CurrentExceptionMessage() << Endl; - return 1; - } - - TBootstrap bootstrap(std::move(options)); - try { - bootstrap.Init(); - bootstrap.Start(); - } catch (...) { - Cerr << CurrentExceptionMessage() << Endl; - bootstrap.Stop(); - return 1; - } - - int exitCode = AppMain(bootstrap); - - try { - bootstrap.Stop(); - } catch (...) { - Cerr << CurrentExceptionMessage() << Endl; - return 1; - } - - return exitCode; -} diff --git a/cloud/filestore/tools/testing/replay/bin/options.cpp b/cloud/filestore/tools/testing/replay/bin/options.cpp deleted file mode 100644 index c7282dd01e..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/options.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include "options.h" - -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -using namespace NLastGetopt; - -//////////////////////////////////////////////////////////////////////////////// - -void TOptions::Parse(int argc, char** argv) -{ - TOpts opts; - opts.AddHelpOption('h'); - - opts.AddLongOption("config", "config file name") - .RequiredArgument("STR") - .StoreResult(&ConfigFile); - - opts.AddLongOption("host", "connect host") - .RequiredArgument("STR") - .StoreResult(&Host); - - opts.AddLongOption("port", "connect port") - .RequiredArgument("NUM") - .StoreResult(&InsecurePort); - - opts.AddLongOption("secure-port", "connect secure port (overrides --port)") - .RequiredArgument("NUM") - .StoreResult(&SecurePort); - - opts.AddLongOption("mon-file") - .RequiredArgument("STR") - .StoreResult(&MonitoringConfig); - - opts.AddLongOption("mon-address") - .RequiredArgument("STR") - .StoreResult(&MonitoringAddress); - - opts.AddLongOption("mon-port") - .RequiredArgument("NUM") - .StoreResult(&MonitoringPort); - - opts.AddLongOption("mon-threads") - .RequiredArgument("NUM") - .StoreResult(&MonitoringThreads); - - opts.AddLongOption("tests-config") - .RequiredArgument("STR") - .StoreResult(&TestsConfig); - - opts.AddLongOption("results") - .RequiredArgument("STR") - .StoreResult(&OutputFile); - - const auto& verbose = opts.AddLongOption("verbose", "output level for diagnostics messages") - .OptionalArgument("STR") - .StoreResult(&VerboseLevel); - - opts.AddLongOption("timeout", "timeout in seconds") - .OptionalArgument("NUM") - .StoreResult(&Timeout); - - opts.AddLongOption("grpc-trace", "turn on grpc tracing") - .NoArgument() - .StoreTrue(&EnableGrpcTracing); - - TOptsParseResultException res(&opts, argc, argv); - - if (res.Has(&verbose) && !VerboseLevel) { - VerboseLevel = "debug"; - } -} - -IOutputStream& TOptions::GetOutputStream() -{ - if (OutputFile && OutputFile != "-") { - if(!OutputStream) { - OutputStream.reset(new TOFStream(OutputFile)); - } - - return *OutputStream; - } - - return Cout; -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/options.h b/cloud/filestore/tools/testing/replay/bin/options.h deleted file mode 100644 index d31beeee58..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/options.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "private.h" - -#include -#include -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct TOptions -{ - TString ConfigFile; - TString Host; - ui32 InsecurePort = 0; - ui32 SecurePort = 0; - - TString MonitoringConfig; - TString MonitoringAddress; - ui32 MonitoringPort = 0; - ui32 MonitoringThreads = 0; - - TString TestsConfig; - - TString OutputFile; - std::unique_ptr OutputStream; - - TDuration Timeout = TDuration::Zero(); - - TString VerboseLevel; - bool EnableGrpcTracing = false; - - void Parse(int argc, char** argv); - - IOutputStream& GetOutputStream(); -}; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/private.h b/cloud/filestore/tools/testing/replay/bin/private.h deleted file mode 100644 index 5dc29567bd..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/private.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -class TBootstrap; - -struct TOptions; -using TOptionsPtr = std::shared_ptr; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/bin/ya.make b/cloud/filestore/tools/testing/replay/bin/ya.make deleted file mode 100644 index fe4eba2bca..0000000000 --- a/cloud/filestore/tools/testing/replay/bin/ya.make +++ /dev/null @@ -1,29 +0,0 @@ -PROGRAM(filestore-replay) - -PEERDIR( - cloud/filestore/config - cloud/filestore/public/api/protos - - cloud/filestore/libs/client - cloud/filestore/tools/testing/replay/lib - cloud/filestore/tools/testing/replay/protos - - cloud/storage/core/libs/common - cloud/storage/core/libs/diagnostics - - contrib/ydb/library/actors/util - library/cpp/getopt - library/cpp/logger - library/cpp/protobuf/json - library/cpp/protobuf/util - library/cpp/sighandler -) - -SRCS( - app.cpp - bootstrap.cpp - main.cpp - options.cpp -) - -END() diff --git a/cloud/filestore/tools/testing/replay/lib/client.cpp b/cloud/filestore/tools/testing/replay/lib/client.cpp deleted file mode 100644 index f679c0d5ec..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/client.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "client.h" diff --git a/cloud/filestore/tools/testing/replay/lib/client.h b/cloud/filestore/tools/testing/replay/lib/client.h deleted file mode 100644 index 917c506f63..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/client.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "public.h" - -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct IClientFactory - : public IStartable -{ - virtual ~IClientFactory() = default; - - virtual IFileStoreServicePtr CreateClient() = 0; -}; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/context.h b/cloud/filestore/tools/testing/replay/lib/context.h deleted file mode 100644 index 2f64b2e974..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/context.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include "public.h" - -#include -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct TAppContext : TNonCopyable -{ - TAtomic ShouldStop = 0; - TAtomic ExitCode = 0; -}; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/executor.cpp b/cloud/filestore/tools/testing/replay/lib/executor.cpp deleted file mode 100644 index 377de67a8b..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/executor.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "executor.h" - -#include "context.h" - -namespace NCloud::NFileStore::NReplay { - -using namespace NThreading; - -//////////////////////////////////////////////////////////////////////////////// - -TExecutor::TExecutor(const TAppContext& ctx, TActionList actions) - : Context(ctx) - , Actions(std::move(actions)) -{ - ThreadPool.Start(); -} - -void TExecutor::Run() -{ - while (!AtomicGet(Context.ShouldStop)) { - ui32 current = -1; - - with_lock (Lock) { - if (DoneCount == Actions.size()) { - break; - } - - current = DoneCount++; - } - - RunAction(current).Subscribe([=] (auto future) { - future.GetValue(); // re-throw exception - ReadyEvent.Signal(); - }); - - ReadyEvent.WaitI(); - ReadyEvent.Reset(); - } -} - -TFuture TExecutor::RunAction(ui32 action) -{ - auto promise = NewPromise(); - auto future = promise.GetFuture(); - - ThreadPool.SafeAddFunc( - [action = std::move(Actions[action]), promise = std::move(promise)] () mutable { - try { - action(); - promise.SetValue(); - } catch (...) { - Y_ABORT("should not happen: %s", CurrentExceptionMessage().c_str()); - } - } - ); - - return future; -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/executor.h b/cloud/filestore/tools/testing/replay/lib/executor.h deleted file mode 100644 index 8cd64c41e7..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/executor.h +++ /dev/null @@ -1,40 +0,0 @@ -#pragma once - -#include "public.h" - -#include - -#include -#include -#include -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -class TExecutor -{ - using TActionList = TVector>; - -private: - const TAppContext& Context; - const TActionList Actions; - - TAdaptiveLock Lock; - TAdaptiveThreadPool ThreadPool; - TManualEvent ReadyEvent; - ui32 DoneCount = 0; - -public: - TExecutor(const TAppContext& ctx, TActionList actions); - - void Run(); - -private: - NThreading::TFuture RunAction(ui32 action); -}; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/public.h b/cloud/filestore/tools/testing/replay/lib/public.h deleted file mode 100644 index 62348a6e68..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/public.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct TAppContext; - -struct IClientFactory; -using IClientFactoryPtr = std::shared_ptr; - -struct IRequestGenerator; -using IRequestGeneratorPtr = std::shared_ptr; - -struct ITest; -using ITestPtr = std::shared_ptr; - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/request.h b/cloud/filestore/tools/testing/replay/lib/request.h deleted file mode 100644 index ecad5a7d97..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/request.h +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once - -#include "public.h" - -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct TCompletedRequest -{ - NProto::EAction Action{}; - TDuration Elapsed; - NProto::TError Error; - bool Stop{}; - TCompletedRequest() = default; - - TCompletedRequest( - NProto::EAction action, - TInstant start, - NProto::TError error) noexcept - : Action(action) - , Elapsed(TInstant::Now() - start) - , Error(std::move(error)) - {} - TCompletedRequest(bool stop) noexcept - : Stop{stop} - {} -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct IRequestGenerator -{ - virtual ~IRequestGenerator() = default; - - virtual bool HasNextRequest() = 0; - // virtual TInstant NextRequestAt() = 0; - virtual NThreading::TFuture ExecuteNextRequest() = 0; - - virtual bool InstantProcessQueue() - { - return false; - } - virtual bool FailOnError() - { - return true; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -IRequestGeneratorPtr CreateReplayRequestGeneratorFs( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - NClient::ISessionPtr session, - TString filesystemId, - NProto::THeaders headers); - -IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - NClient::ISessionPtr session, - TString filesystemId, - NProto::THeaders headers); - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/test.cpp b/cloud/filestore/tools/testing/replay/lib/test.cpp deleted file mode 100644 index 151076c6ec..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/test.cpp +++ /dev/null @@ -1,1156 +0,0 @@ -#include "test.h" - -#include "client.h" -#include "context.h" -#include "request.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace NCloud::NFileStore::NReplay { - -using namespace NThreading; -using namespace NCloud::NFileStore::NClient; - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -struct TTestStats -{ - struct TStats - { - ui64 Requests = 0; - TLatencyHistogram Hist; - }; - - TString Name; - bool Success = true; - TMap ActionStats; -}; - -//////////////////////////////////////////////////////////////////////////////// - -template -T WaitForCompletion(const TString& request, const TFuture& future) -{ - const auto& response = future.GetValue(TDuration::Max()); - if (HasError(response)) { - const auto& error = response.GetError(); - throw yexception() << "Failed to execute " << request - << " request: " << FormatError(error); - } - - return response; -} - -//////////////////////////////////////////////////////////////////////////////// - -class TRequestsCompletionQueue -{ -private: - TAdaptiveLock Lock; - TDeque> Items; - -public: - void Enqueue(std::unique_ptr request) - { - with_lock (Lock) { - Items.emplace_back(std::move(request)); - } - } - - std::unique_ptr Dequeue() - { - with_lock (Lock) { - std::unique_ptr ptr; - if (Items) { - ptr = std::move(Items.front()); - Items.pop_front(); - } - - return ptr; - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct TSetupSession -{ - bool ReadOnly = false; - ui64 SessionSeqNo = 0; - TPromise Complete; -}; - -struct TTeardown -{ - TPromise Complete; -}; -struct TSuspendLoad -{ -}; -struct TResumeLoad -{ -}; -struct TProcessCompletedRequests -{ -}; - -using TLoadTestCommand = std::variant< - TTeardown, - TSuspendLoad, - TResumeLoad, - TProcessCompletedRequests, - TSetupSession>; - -//////////////////////////////////////////////////////////////////////////////// - -class TLoadTestCommandChannel -{ -private: - TAdaptiveLock Lock; - TDeque Items; - bool HasProcessCompletedRequests = false; - -public: - void Enqueue(TLoadTestCommand command) - { - with_lock (Lock) { - if (holds_alternative(command)) { - if (HasProcessCompletedRequests) { - return; - } - HasProcessCompletedRequests = true; - } - Items.emplace_back(std::move(command)); - } - } - - TMaybe Dequeue() - { - with_lock (Lock) { - if (Items) { - auto value = std::move(Items.front()); - Items.pop_front(); - if (holds_alternative(value)) { - HasProcessCompletedRequests = false; - } - - return value; - } - - return {}; - } - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct TStartSource -{ -}; - -struct TStartTarget -{ -}; - -struct TFinishMigration -{ -}; - -struct TTestFinished -{ - TString Id; -}; - -using TLoadTestControllerCommand = - std::variant; - -//////////////////////////////////////////////////////////////////////////////// - -class TLoadTestControllerCommandChannel -{ -private: - TAdaptiveLock Lock; - TDeque Items; - TAutoEvent Event; - -public: - void Enqueue(TLoadTestControllerCommand command) - { - with_lock (Lock) { - Items.emplace_back(std::move(command)); - } - Event.Signal(); - } - - TMaybe Dequeue() - { - with_lock (Lock) { - if (Items) { - auto value = std::move(Items.front()); - Items.pop_front(); - return value; - } - - return {}; - } - } - - void Wait() - { - Event.WaitI(); - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -class TLoadTest final - : public ITest - , public ISimpleThread - , public std::enable_shared_from_this -{ -private: - static constexpr TDuration ReportInterval = TDuration::Seconds(5); - - const NProto::TLoadTest Config; - const ITimerPtr Timer; - const ISchedulerPtr Scheduler; - const ILoggingServicePtr Logging; - const TString TestInstanceId; - const TString TestTag; - - TLoadTestControllerCommandChannel& Controller; - - TLog Log; - IFileStoreServicePtr Client; - ISessionPtr Session; - - TString FileSystemId; - TString ClientId; - TString SessionId; - - ui32 MaxIoDepth = 64; - ui32 CurrentIoDepth = 0; - - TDuration MaxDuration; - TInstant StartTs; - - ui64 MaxRequests = 0; - ui64 RequestsSent = 0; - ui64 RequestsCompleted = 0; - - TInstant LastReportTs; - ui64 LastRequestsCompleted = 0; - - IRequestGeneratorPtr RequestGenerator; - TRequestsCompletionQueue CompletionQueue; - TLoadTestCommandChannel Commands; - TAutoEvent Event; - - bool LoadEnabled = true; - - ui64 SeqNo = 0; - - TTestStats TestStats; - TPromise Result = NewPromise(); - - bool ShutdownFlag = false; - -public: - TLoadTest( - const NProto::TLoadTest& config, - ITimerPtr timer, - ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IClientFactoryPtr clientFactory, - TString testInstanceId, - TLoadTestControllerCommandChannel& controller, - TString clientId = {}) - : Config(config) - , Timer(std::move(timer)) - , Scheduler(std::move(scheduler)) - , Logging(std::move(logging)) - , TestInstanceId(std::move(testInstanceId)) - , TestTag(NReplay::MakeTestTag(Config.GetName())) - , Controller(controller) - , Client(clientFactory->CreateClient()) - , ClientId(std::move(clientId)) - { - Log = Logging->CreateLog("RUNNER"); - } - - TFuture Run() override - { - ISimpleThread::Start(); - return Result; - } - - void GenerateRequests() - { - if (LoadEnabled && !ShouldStop()) { - while (SendNextRequest()) { - // DUMP("rb", RequestsSent); - ++RequestsSent; - STORAGE_DEBUG( - "%s request sent: %s %lu %lu", - MakeTestTag().c_str(), - SessionId.c_str(), - SeqNo, - RequestsSent); - } - } - } - - void EnqueueCommand(TLoadTestCommand command) - { - Commands.Enqueue(std::move(command)); - Event.Signal(); - } - - void SetupSession(TSetupSession& cmd) - { - if (!ShouldStop()) { - STORAGE_INFO( - "%s establishing session %lu", - MakeTestTag().c_str(), - cmd.SessionSeqNo); - - TFuture result; - if (!SessionId) { - result = Session->CreateSession(cmd.ReadOnly, cmd.SessionSeqNo); - } else { - result = Session->AlterSession(cmd.ReadOnly, cmd.SessionSeqNo); - } - - WaitForCompletion("create-session", result); - auto response = result.GetValue(); - if (FAILED(response.GetError().GetCode())) { - STORAGE_INFO( - "%s failed establish session: %s %lu %s", - MakeTestTag().c_str(), - SessionId.c_str(), - cmd.SessionSeqNo, - response.GetError().GetMessage().c_str()); - TestStats.Success = false; - cmd.Complete.SetValue(false); - return; - } - auto sessionId = response.GetSession().GetSessionId(); - if (SessionId && SessionId != sessionId) { - STORAGE_INFO( - "%s session established but id has changed: %s %lu %s", - MakeTestTag().c_str(), - SessionId.c_str(), - cmd.SessionSeqNo, - sessionId.c_str()); - TestStats.Success = false; - cmd.Complete.SetValue(false); - return; - } - SessionId = sessionId; - STORAGE_INFO( - "%s session established: %s %lu", - MakeTestTag().c_str(), - SessionId.c_str(), - cmd.SessionSeqNo); - } - - cmd.Complete.SetValue(true); - } - - bool HandleTeardown(TTeardown& cmd) - { - STORAGE_INFO("%s got teardown", MakeTestTag().c_str()) - TeardownTest(cmd); - return false; - } - - bool HandleSuspendLoad(const TSuspendLoad&) - { - LoadEnabled = false; - return true; - } - - bool HandleResumeLoad(const TResumeLoad&) - { - LoadEnabled = true; - GenerateRequests(); - return true; - } - - bool HandleProcessCompletedRequests(const TProcessCompletedRequests&) - { - ProcessCompletedRequests(); - ReportProgress(); - GenerateRequests(); - return true; - } - - bool HandleSetupSession(TSetupSession& cmd) - { - SeqNo = cmd.SessionSeqNo; - SetupSession(cmd); - GenerateRequests(); - return true; - } - - bool HandleCommand(TLoadTestCommand cmd) - { - return std::visit( - TOverloaded{ - [this](TTeardown& cmd) { return HandleTeardown(cmd); }, - [this](const TSuspendLoad& cmd) - { return HandleSuspendLoad(cmd); }, - [this](const TResumeLoad& cmd) - { return HandleResumeLoad(cmd); }, - [this](const TProcessCompletedRequests& cmd) - { return HandleProcessCompletedRequests(cmd); }, - [this](TSetupSession& cmd) - { - return HandleSetupSession(cmd); - }}, - cmd); - } - - void* ThreadProc() override - { - StartTs = TInstant::Now(); - NCloud::SetCurrentThreadName(MakeTestTag()); - - try { - SetupTest(); - LastReportTs = TInstant::Now(); - - for (;;) { - Event.WaitI(); - bool cont = true; - while (auto maybeCmd = Commands.Dequeue()) { - auto& cmd = *maybeCmd; - if (cont = HandleCommand(cmd); !cont) { - break; - } - } - if (ShouldStop() && !ShutdownFlag) { - TTeardown dummy; - TeardownTest(dummy); - - TTestFinished cmd; - cmd.Id = TestInstanceId; - Controller.Enqueue(cmd); - ShutdownFlag = true; - } - if (!cont) { - break; - } - } - - // prevent race between this thread and main thread - // destroying test instance right after setvalue() - auto result = Result; - result.SetValue(GetStats()); - } catch (...) { - STORAGE_ERROR( - "%s test has failed: %s", - MakeTestTag().c_str(), - CurrentExceptionMessage().c_str()); - - Result.SetException(std::current_exception()); - } - - return nullptr; - } - - TString GetTestId() const - { - return TestInstanceId; - } - -private: - void SetupTest() - { - Client->Start(); - - // setup test limits - if (auto depth = Config.GetIODepth()) { - MaxIoDepth = depth; - } - - MaxRequests = Config.GetRequestsCount(); - MaxDuration = TDuration::Seconds(Config.GetTestDuration()); - - STORAGE_INFO("%s setting up test", MakeTestTag().c_str()); - - // setup test filestore - if (Config.HasFileSystemId()) { - FileSystemId = Config.GetFileSystemId(); - } else if (Config.HasCreateFileStoreRequest()) { - FileSystemId = Config.GetCreateFileStoreRequest().GetFileSystemId(); - } - - CreateSession(); - - NProto::THeaders headers; - headers.SetClientId(Config.GetName()); - headers.SetSessionId(SessionId); - - switch (Config.GetSpecsCase()) { - case NProto::TLoadTest::kReplayFsSpec: - RequestGenerator = CreateReplayRequestGeneratorFs( - Config.GetReplayFsSpec(), - Logging, - Session, - FileSystemId, - headers); - break; - - case NProto::TLoadTest::kReplayGrpcSpec: - RequestGenerator = CreateReplayRequestGeneratorGRPC( - Config.GetReplayGrpcSpec(), - Logging, - Session, - FileSystemId, - headers); - break; - - default: - ythrow yexception() - << MakeTestTag() << " config should have test spec"; - } - } - - void CreateSession() - { - NProto::TSessionConfig proto; - proto.SetFileSystemId(FileSystemId); - proto.SetClientId(ClientId); - proto.SetSessionPingTimeout(Config.GetSessionPingTimeout()); - proto.SetSessionRetryTimeout(Config.GetSessionRetryTimeout()); - - Session = NClient::CreateSession( - Logging, - Timer, - Scheduler, - Client, - std::make_shared(proto)); - } - - bool ShouldStop() const - { - return !TestStats.Success || LimitsReached(); - } - - bool LimitsReached() const - { - return (MaxDuration && TInstant::Now() - StartTs >= MaxDuration) || - (MaxRequests && RequestsSent >= MaxRequests); - } - - bool SendNextRequest() - { - if (LimitsReached() || (MaxIoDepth && CurrentIoDepth >= MaxIoDepth) || - !RequestGenerator->HasNextRequest() || ShouldStop()) - { - return false; - } - - auto self = weak_from_this(); - ++CurrentIoDepth; - const auto future = RequestGenerator->ExecuteNextRequest().Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - ptr->SignalCompletion(future.GetValue()); - } - }); - - if (RequestGenerator->InstantProcessQueue()) { - if (future.HasValue()) { - ProcessCompletedRequests(); - } - } - - return true; - } - - void SignalCompletion(TCompletedRequest request) - { - CompletionQueue.Enqueue( - std::make_unique(std::move(request))); - - Commands.Enqueue(TProcessCompletedRequests{}); - Event.Signal(); - } - - void ProcessCompletedRequests() - { - while (auto request = CompletionQueue.Dequeue()) { - Y_ABORT_UNLESS(CurrentIoDepth > 0); - --CurrentIoDepth; - ++RequestsCompleted; - - auto code = request->Error.GetCode(); - if (FAILED(code)) { - STORAGE_WARN( - "%s failed request %d: %s", - MakeTestTag().c_str(), - request->Action, - FormatError(request->Error).c_str()); - - if (RequestGenerator->FailOnError()) { - TestStats.Success = false; - } - } - - if (request->Stop) { - TestStats.Success = false; - } - auto& stats = TestStats.ActionStats[request->Action]; - ++stats.Requests; - stats.Hist.RecordValue(request->Elapsed); - } - // DUMP("compl"); - } - - void ReportProgress() - { - auto now = TInstant::Now(); - auto elapsed = now - LastReportTs; - - if (elapsed > ReportInterval) { - const auto requestsCompleted = - RequestsCompleted - LastRequestsCompleted; - - auto stats = GetStats(); - STORAGE_INFO( - "%s current rate: %ld r/s; stats:\n%s", - MakeTestTag().c_str(), - (ui64)(requestsCompleted / elapsed.Seconds()), - NProtobufJson::Proto2Json(stats, {.FormatOutput = true}) - .c_str()); - - LastReportTs = now; - LastRequestsCompleted = RequestsCompleted; - } - } - - NProto::TTestStats GetStats() - { - NProto::TTestStats results; - results.SetName(Config.GetName()); - results.SetSuccess(TestStats.Success); - - auto* stats = results.MutableStats(); - for (const auto& pair: TestStats.ActionStats) { - auto* action = stats->Add(); - action->SetAction(NProto::EAction_Name(pair.first)); - action->SetCount(pair.second.Requests); - FillLatency(pair.second.Hist, *action->MutableLatency()); - } - - // TODO report some stats for the files generated during the shooting - - return results; - } - - void TeardownTest(TTeardown& cmd) - { - if (ShutdownFlag) { - if (cmd.Complete.Initialized()) { - cmd.Complete.SetValue(true); - } - return; - } - - STORAGE_INFO("%s tear down", MakeTestTag().c_str()); - while (CurrentIoDepth > 0) { - Sleep(TDuration::Seconds(1)); - ProcessCompletedRequests(); - } - auto ctx = MakeIntrusive(); - - auto result = Session->DestroySession(); - WaitForCompletion("destroy session", result); - - if (Client) { - Client->Stop(); - } - - auto response = result.GetValue(); - if (FAILED(response.GetError().GetCode())) { - STORAGE_INFO( - "%s failed to destroy session: %s %lu %s", - MakeTestTag().c_str(), - SessionId.c_str(), - SeqNo, - response.GetError().GetMessage().c_str()); - TestStats.Success = false; - } - - if (cmd.Complete.Initialized()) { - cmd.Complete.SetValue(SUCCEEDED(response.GetError().GetCode())); - } - } - - void FillLatency(const TLatencyHistogram& hist, NProto::TLatency& latency) - { - latency.SetP50(hist.GetValueAtPercentile(50)); - latency.SetP90(hist.GetValueAtPercentile(90)); - latency.SetP95(hist.GetValueAtPercentile(95)); - latency.SetP99(hist.GetValueAtPercentile(99)); - latency.SetP999(hist.GetValueAtPercentile(99.9)); - latency.SetMin(hist.GetMin()); - latency.SetMax(hist.GetMax()); - latency.SetMean(hist.GetMean()); - latency.SetStdDeviation(hist.GetStdDeviation()); - } - - const TString& MakeTestTag() const - { - return TestTag; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - -std::shared_ptr CreateSimpleLoadTest( - const NProto::TLoadTest& config, - ITimerPtr timer, - ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IClientFactoryPtr clientFactory, - TString testInstanceId, - TLoadTestControllerCommandChannel& controller, - TString clientId) -{ - return std::make_shared( - config, - std::move(timer), - std::move(scheduler), - std::move(logging), - std::move(clientFactory), - std::move(testInstanceId), - controller, - std::move(clientId)); -} - -//////////////////////////////////////////////////////////////////////////////// - -class TLoadTestController final - : public ITest - , public ISimpleThread - , public std::enable_shared_from_this -{ -private: - const TAppContext& Ctx; - const NProto::TLoadTest Config; - const ITimerPtr Timer; - const ISchedulerPtr Scheduler; - const ILoggingServicePtr Logging; - const IClientFactoryPtr ClientFactory; - const TString TestTag; - - TLog Log; - std::shared_ptr SourceTest; - std::shared_ptr TargetTest; - TFuture SourceFuture; - TFuture TargetFuture; - - TDuration MaxDuration; - - TInstant StartTs; - - TLoadTestControllerCommandChannel Events; - - TTestStats TestStats; - TPromise Result = NewPromise(); - - ui64 SessionSeqNo = 0; - - IFileStoreServicePtr Client; - -public: - TLoadTestController( - const TAppContext& ctx, - const NProto::TLoadTest& config, - ITimerPtr timer, - ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IClientFactoryPtr clientFactory) - : Ctx(ctx) - , Config(config) - , Timer(std::move(timer)) - , Scheduler(std::move(scheduler)) - , Logging(std::move(logging)) - , ClientFactory(std::move(clientFactory)) - , TestTag(NReplay::MakeTestTag("Controller_" + Config.GetName())) - , Client(ClientFactory->CreateClient()) - { - Log = Logging->CreateLog("TEST_CONTROLLER"); - } - - TFuture Run() override - { - Events.Enqueue(TStartSource{}); - - ISimpleThread::Start(); - - return Result; - } - - TFuture SendSetupTestSession( - std::shared_ptr& test, - bool readOnly, - ui64 sessionSeqNo) - { - TPromise complete = NewPromise(); - - TSetupSession cmd; - cmd.ReadOnly = readOnly; - cmd.SessionSeqNo = sessionSeqNo; - cmd.Complete = complete; - test->EnqueueCommand(std::move(cmd)); - - return complete.GetFuture(); - } - - TFuture SendTeardownTest(std::shared_ptr& test) - { - TPromise complete = NewPromise(); - TTeardown teardown; - teardown.Complete = complete; - test->EnqueueCommand(teardown); - - return complete.GetFuture(); - } - - bool HandleStartSource(const TStartSource&) - { - auto testId = CreateGuidAsString(); - STORAGE_ERROR( - "%s start test %s", - MakeTestTag().c_str(), - testId.c_str()); - - SourceTest = CreateSimpleLoadTest( - Config, - Timer, - Scheduler, - Logging, - ClientFactory, - std::move(testId), - Events, - "migration-test"); - SourceFuture = SourceTest->Run(); - - auto result = SendSetupTestSession(SourceTest, false, SessionSeqNo); - if (!result.GetValueSync()) { - return false; - } - - auto period = Config.GetMigrationSpec().GetMigrationPeriod(); - if (period) { - Scheduler->Schedule( - Timer->Now() + TDuration::Seconds(period), - [weakPtr = weak_from_this()] - { - if (auto self = weakPtr.lock()) { - self->Events.Enqueue(TStartTarget{}); - } - }); - } - return true; - } - - bool HandleStartTarget(const TStartTarget&) - { - auto testId = CreateGuidAsString(); - STORAGE_INFO( - "%s start a new client %s", - MakeTestTag().c_str(), - testId.c_str()); - - TargetTest = CreateSimpleLoadTest( - Config, - Timer, - Scheduler, - Logging, - ClientFactory, - std::move(testId), - Events, - "migration-test"); - TargetFuture = TargetTest->Run(); - - auto result = SendSetupTestSession(TargetTest, true, SessionSeqNo + 1); - if (!result.GetValueSync()) { - return false; - } - - auto period = Config.GetMigrationSpec().GetStateTransferDelay(); - if (period) { - Scheduler->Schedule( - Timer->Now() + TDuration::Seconds(period), - [weakPtr = weak_from_this()] - { - if (auto self = weakPtr.lock()) { - self->Events.Enqueue(TFinishMigration{}); - } - }); - } - return true; - } - - bool HandleFinishMigration(const TFinishMigration&) - { - STORAGE_INFO( - "%s switch to new client %s", - MakeTestTag().c_str(), - TargetTest->GetTestId().c_str()); - - auto source = SendSetupTestSession(SourceTest, true, SessionSeqNo); - auto target = SendSetupTestSession(TargetTest, false, ++SessionSeqNo); - - TVector> futures{source, target}; - WaitAll(futures).Wait(); - if (!source.GetValue() || !target.GetValue()) { - STORAGE_ERROR( - "%s failed to switch to new client", - MakeTestTag().c_str()); - return false; - } - - TTeardown teardown; - SourceTest->EnqueueCommand(teardown); - SourceFuture.GetValueSync(); - - SourceFuture = - std::exchange(TargetFuture, TFuture()); - std::swap(SourceTest, TargetTest); - TargetTest.reset(); - - auto period = Config.GetMigrationSpec().GetMigrationPeriod(); - Scheduler->Schedule( - Timer->Now() + TDuration::Seconds(period), - [weakPtr = weak_from_this()] - { - if (auto self = weakPtr.lock()) { - self->Events.Enqueue(TStartTarget()); - } - }); - return true; - } - - bool HandleTestFinished(const TTestFinished& cmd) - { - STORAGE_INFO( - "%s test finished %s", - MakeTestTag().c_str(), - cmd.Id.c_str()); - - if (SourceTest && SourceTest->GetTestId() == cmd.Id) { - return false; - } - - if (TargetTest && TargetTest->GetTestId() == cmd.Id) { - return false; - } - - return true; - } - - bool HandleStateChange(TLoadTestControllerCommand cmd) - { - return std::visit( - TOverloaded{ - [this](const TStartSource& cmd) - { return HandleStartSource(cmd); }, - [this](const TStartTarget& cmd) - { return HandleStartTarget(cmd); }, - [this](const TFinishMigration& cmd) - { return HandleFinishMigration(cmd); }, - [this](const TTestFinished& cmd) - { - return HandleTestFinished(cmd); - }}, - cmd); - } - - void SetupTest() - { - Client->Start(); - - MaxDuration = TDuration::Seconds(Config.GetTestDuration()); - - if (Config.HasFileSystemId() && Config.HasCreateFileStoreRequest()) { - ythrow yexception() << MakeTestTag() - << " config should have either existing " - "filesystem id or request to create one"; - } - - if (Config.HasCreateFileStoreRequest()) { - auto request = std::make_shared( - Config.GetCreateFileStoreRequest()); - - STORAGE_INFO( - "%s create filestore: %s", - MakeTestTag().c_str(), - DumpMessage(*request).c_str()); - - TCallContextPtr ctx = MakeIntrusive(); - auto result = Client->CreateFileStore(ctx, request); - WaitForCompletion(GetRequestName(*request), result); - } - } - - void CleanupTest() - { - if (SourceTest) { - auto result = SendTeardownTest(SourceTest).GetValueSync(); - if (!result) { - STORAGE_ERROR( - "%s failed to teardown source client", - MakeTestTag().c_str()); - } - } - - if (TargetTest) { - auto result = SendTeardownTest(TargetTest).GetValueSync(); - if (!result) { - STORAGE_ERROR( - "%s failed to teardown source client", - MakeTestTag().c_str()); - } - TargetFuture.GetValueSync(); - } - - if (Config.HasCreateFileStoreRequest() && !Config.GetKeepFileStore()) { - auto filesystemId = - Config.GetCreateFileStoreRequest().GetFileSystemId(); - - STORAGE_INFO( - "%s destroy fs %s", - MakeTestTag().c_str(), - filesystemId.c_str()); - - auto request = std::make_shared(); - request->SetFileSystemId(filesystemId); - - TCallContextPtr ctx = MakeIntrusive(); - auto result = Client->DestroyFileStore(ctx, request); - WaitForCompletion(GetRequestName(*request), result); - } - - Client->Stop(); - - auto result = Result; - result.SetValue(SourceFuture.GetValueSync()); - } - - void* ThreadProc() override - { - StartTs = TInstant::Now(); - NCloud::SetCurrentThreadName(MakeTestTag()); - - SetupTest(); - - while (!ShouldStop()) { - try { - Events.Wait(); - bool cont = true; - while (auto maybeCmd = Events.Dequeue()) { - auto& cmd = *maybeCmd; - cont = HandleStateChange(cmd); - if (!cont) { - break; - } - } - if (!cont) { - break; - } - } catch (...) { - STORAGE_ERROR( - "%s test has failed: %s", - MakeTestTag().c_str(), - CurrentExceptionMessage().c_str()); - - Result.SetException(std::current_exception()); - } - } - - STORAGE_INFO("test is going to finish"); - - CleanupTest(); - - return nullptr; - } - -private: - bool ShouldStop() const - { - return AtomicGet(Ctx.ShouldStop) || - (MaxDuration && TInstant::Now() - StartTs >= MaxDuration) || - (TargetFuture.HasValue() || SourceFuture.HasValue()); - } - - const TString& MakeTestTag() const - { - return TestTag; - } -}; - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -TString MakeTestTag(const TString& name) -{ - return Sprintf("[%s]", name.c_str()); -} - -ITestPtr CreateLoadTest( - const TAppContext& ctx, - const NProto::TLoadTest& config, - ITimerPtr timer, - ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IClientFactoryPtr clientFactory) -{ - return std::make_shared( - ctx, - config, - std::move(timer), - std::move(scheduler), - std::move(logging), - std::move(clientFactory)); -} - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/test.h b/cloud/filestore/tools/testing/replay/lib/test.h deleted file mode 100644 index e4f9a7fcaf..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/test.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "public.h" - -#include - -#include -#include - -#include - -#include -#include - -namespace NCloud::NFileStore::NReplay { - -//////////////////////////////////////////////////////////////////////////////// - -struct ITest -{ - virtual ~ITest() = default; - - virtual NThreading::TFuture Run() = 0; -}; - -//////////////////////////////////////////////////////////////////////////////// - -TString MakeTestTag(const TString& name); - -ITestPtr CreateLoadTest( - const TAppContext& ctx, - const NProto::TLoadTest& config, - ITimerPtr timer, - ISchedulerPtr scheduler, - ILoggingServicePtr logging, - IClientFactoryPtr clientFactory); - -} // namespace NCloud::NFileStore::NReplay diff --git a/cloud/filestore/tools/testing/replay/lib/ya.make b/cloud/filestore/tools/testing/replay/lib/ya.make deleted file mode 100644 index ea59021d8d..0000000000 --- a/cloud/filestore/tools/testing/replay/lib/ya.make +++ /dev/null @@ -1,36 +0,0 @@ -LIBRARY(filestore-testing-replay-lib) - -SRCS( - client.cpp - context.h - executor.cpp - request_replay_fs.cpp - request_replay_grpc.cpp - test.cpp -) - -PEERDIR( - cloud/filestore/config - cloud/filestore/public/api/protos - - cloud/filestore/libs/client - cloud/filestore/libs/service - - cloud/storage/core/libs/common - cloud/storage/core/libs/diagnostics - - cloud/filestore/tools/testing/replay/protos - - library/cpp/deprecated/atomic - - library/cpp/eventlog - cloud/filestore/libs/diagnostics/events - cloud/filestore/libs/service_local - cloud/filestore/tools/analytics/libs/event-log - library/cpp/testing/unittest - - library/cpp/aio -) - - -END() diff --git a/cloud/filestore/tools/testing/replay/protos/replay.proto b/cloud/filestore/tools/testing/replay/protos/replay.proto deleted file mode 100644 index 025e3f6392..0000000000 --- a/cloud/filestore/tools/testing/replay/protos/replay.proto +++ /dev/null @@ -1,126 +0,0 @@ -syntax = "proto3"; - -package NCloud.NFileStore.NProto; - -import "cloud/filestore/public/api/protos/fs.proto"; - -//////////////////////////////////////////////////////////////////////////////// - - -enum EAction -{ - // INDEX - ACTION_CREATE_NODE = 0; - ACTION_RENAME_NODE = 1; - ACTION_REMOVE_NODE = 2; - ACTION_CREATE_HANDLE = 3; - ACTION_DESTROY_HANDLE = 4; - // FIXME: should allocate data before locking - ACTION_ACQUIRE_LOCK = 5; - ACTION_RELEASE_LOCK = 6; - ACTION_GET_NODE_ATTR = 7; - ACTION_ACCESS_NODE = 8; - ACTION_LIST_NODES = 9; - - // DATA - ACTION_WRITE = 101; - ACTION_READ = 102; -}; - - -message TReplaySpec -{ - string FileName = 1; - //bool ValidationEnabled = 6; - //bool Sequential = 7; - string ReplayRoot = 8; // not used in grpc - bool NoRead = 9; - bool NoWrite = 10; - bool Prepare = 11; // Only write files needed for reading - double TimeScale = 12; - bool WriteRandom = 13; - bool WriteEmpty = 14; - bool CreateOnRead = 15; -} - -message TMigrationSpec -{ - uint32 MigrationPeriod = 1; - uint32 StateTransferDelay = 2; -} - -message TLoadTest -{ - string Name = 1; - - oneof FileStore - { - // Existing filestore name - string FileSystemId = 2; - - // Create a new one - TCreateFileStoreRequest CreateFileStoreRequest = 3; - } - - oneof Specs - { - TReplaySpec ReplayFsSpec = 17; - TReplaySpec ReplayGrpcSpec = 18; - } - - uint32 IODepth = 10; - uint32 RequestsCount = 11; - uint32 TestDuration = 12; - uint32 SessionPingTimeout = 13; - uint32 SessionRetryTimeout = 14; - - TMigrationSpec MigrationSpec = 15; - - // disables destruction of the filestore created via CreateFileStoreRequest - bool KeepFileStore = 16; -} - -//////////////////////////////////////////////////////////////////////////////// - -message TLatency -{ - uint64 P50 = 1; - uint64 P95 = 2; - uint64 P90 = 3; - uint64 P99 = 4; - uint64 P999 = 5; - uint64 Min = 6; - uint64 Max = 7; - double Mean = 8; - double StdDeviation = 9; -} - -message TTestStats -{ - message TStats - { - string Action = 1; - uint64 Count = 2; - TLatency Latency = 3; - } - - string Name = 1; - bool Success = 2; - - repeated TStats Stats = 3; -} - -//////////////////////////////////////////////////////////////////////////////// - -message TTestGraph -{ - message TTest - { - oneof Test - { - TLoadTest LoadTest = 1; - } - } - - repeated TTest Tests = 1; -} diff --git a/cloud/filestore/tools/testing/replay/protos/ya.make b/cloud/filestore/tools/testing/replay/protos/ya.make deleted file mode 100644 index 65814148f7..0000000000 --- a/cloud/filestore/tools/testing/replay/protos/ya.make +++ /dev/null @@ -1,13 +0,0 @@ -PROTO_LIBRARY(filestore-testing-replay-protos) - -ONLY_TAGS(CPP_PROTO) - -PEERDIR( - cloud/filestore/public/api/protos -) - -SRCS( - replay.proto -) - -END() diff --git a/cloud/filestore/tools/testing/replay/ya.make b/cloud/filestore/tools/testing/replay/ya.make deleted file mode 100644 index 908f2c16ef..0000000000 --- a/cloud/filestore/tools/testing/replay/ya.make +++ /dev/null @@ -1,5 +0,0 @@ -RECURSE( - bin - lib - protos -) From 7b4c62b8756ad45471c4eb06c67ae96b475279a0 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 25 Sep 2024 18:07:00 +0000 Subject: [PATCH 10/44] move fix --- cloud/filestore/tools/testing/loadtest/lib/test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index b930702695..3242f744b1 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -594,6 +594,8 @@ class TLoadTest final ++CurrentIoDepth; auto self = weak_from_this(); const auto future = RequestGenerator->ExecuteNextRequest().Apply( + [=](const TFuture& future) + { if (auto ptr = self.lock()) { ptr->SignalCompletion(future.GetValue()); } From ff0574b2bb43895516c792bb0129c10c0c34a94d Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 25 Sep 2024 19:26:53 +0000 Subject: [PATCH 11/44] better --- .../loadtest/lib/request_replay_fs.cpp | 328 ++++++++++-------- .../loadtest/lib/request_replay_grpc.cpp | 207 ++++++----- 2 files changed, 285 insertions(+), 250 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index dd19fad252..292f660b5c 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -237,36 +237,39 @@ class TReplayRequestGeneratorFs final NThreading::TFuture ExecuteNextRequest() override { if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest{}); + return MakeFuture(TCompletedRequest(true)); } + for (; EventPtr; Advance()) { if (!EventLogMessagePtr) { continue; } - STORAGE_DEBUG("Processing %d", EventMessageNumber); for (; EventMessageNumber > 0;) { auto request = EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - TimestampMcs = request.GetTimestampMcs(); - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - if (timediff > diff.MicroSeconds()) { - auto slp = - TDuration::MicroSeconds(timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); + { + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = TDuration::MicroSeconds( + timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); - Sleep(slp); + Sleep(slp); + } + Started = current; } - Started = current; STORAGE_DEBUG( "Processing message " @@ -329,39 +332,38 @@ class TReplayRequestGeneratorFs final private: TFuture DoAccessNode( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} - auto started = TInstant::Now(); - if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto node = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + const auto node = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); if (!node) { STORAGE_ERROR( - "access fail: " << " no node=" << r.GetNodeInfo().GetNodeId()); + "access fail: " << " no node=" + << logRequest.GetNodeInfo().GetNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, - started, + Started, MakeError(E_FAIL, "cancelled")}); } auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); int res = access(fname.c_str(), R_OK); STORAGE_DEBUG( - "access " << node << " <- " << r.GetNodeInfo().GetNodeId() << " = " - << res); + "access " << node << " <- " << logRequest.GetNodeInfo().GetNodeId() + << " = " << res); return MakeFuture( - TCompletedRequest{NProto::ACTION_ACCESS_NODE, started, {}}); + TCompletedRequest{NProto::ACTION_ACCESS_NODE, Started, {}}); } // Recursive, no infinity loop check @@ -421,7 +423,7 @@ class TReplayRequestGeneratorFs final } TFuture DoCreateHandle( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} @@ -430,69 +432,73 @@ class TReplayRequestGeneratorFs final // handle=11024287581389312, size=0} TGuard guard(StateLock); - auto started = TInstant::Now(); TString relativePathName; - if (r.GetNodeInfo().GetNodeId()) { - if (auto path = PathByNode(r.GetNodeInfo().GetNodeId())) { + if (logRequest.GetNodeInfo().GetNodeId()) { + if (auto path = PathByNode(logRequest.GetNodeInfo().GetNodeId())) { relativePathName = path; } } if (relativePathName.empty()) { - auto parentNode = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + auto parentNode = + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!parentNode) { parentNode = NodeIdMapped( - KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].ParentLog); + KnownLogNodes[logRequest.GetNodeInfo().GetParentNodeId()] + .ParentLog); } - if (!parentNode && r.GetNodeInfo().GetParentNodeId() != - r.GetNodeInfo().GetNodeId()) + if (!parentNode && logRequest.GetNodeInfo().GetParentNodeId() != + logRequest.GetNodeInfo().GetNodeId()) { parentNode = CreateDirIfMissingByNodeLog( - r.GetNodeInfo().GetParentNodeId()); + logRequest.GetNodeInfo().GetParentNodeId()); } if (!parentNode) { STORAGE_ERROR( "create handle fail :" - << r.GetNodeInfo().GetHandle() - << " no parent=" << r.GetNodeInfo().GetParentNodeId()); + << logRequest.GetNodeInfo().GetHandle() << " no parent=" + << logRequest.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, - started, + Started, MakeError(E_FAIL, "cancelled")}); } - auto nodeName = r.GetNodeInfo().GetNodeName(); - if (nodeName.empty() && r.GetNodeInfo().GetNodeId() != - r.GetNodeInfo().GetParentNodeId()) + auto nodeName = logRequest.GetNodeInfo().GetNodeName(); + if (nodeName.empty() && + logRequest.GetNodeInfo().GetNodeId() != + logRequest.GetNodeInfo().GetParentNodeId()) { - nodeName = KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name; + nodeName = + KnownLogNodes[logRequest.GetNodeInfo().GetNodeId()].Name; } auto parentpath = PathByNode(parentNode); if (nodeName.empty() && IsDir(Spec.GetReplayRoot() + parentpath)) { nodeName = - KnownLogNodes[r.GetNodeInfo().GetParentNodeId()].Name; + KnownLogNodes[logRequest.GetNodeInfo().GetParentNodeId()] + .Name; } relativePathName = parentpath + nodeName; } STORAGE_DEBUG( "open " << relativePathName - << " handle=" << r.GetNodeInfo().GetHandle() - << " flags=" << r.GetNodeInfo().GetFlags() << " " - << HandleFlagsToString(r.GetNodeInfo().GetFlags()) - << " mode=" << r.GetNodeInfo().GetMode() - << " node=" << r.GetNodeInfo().GetNodeId()); + << " handle=" << logRequest.GetNodeInfo().GetHandle() + << " flags=" << logRequest.GetNodeInfo().GetFlags() << " " + << HandleFlagsToString(logRequest.GetNodeInfo().GetFlags()) + << " mode=" << logRequest.GetNodeInfo().GetMode() + << " node=" << logRequest.GetNodeInfo().GetNodeId()); try { TFile fileHandle( Spec.GetReplayRoot() + relativePathName, // OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr) FileOpenFlags( - r.GetNodeInfo().GetFlags(), + logRequest.GetNodeInfo().GetFlags(), Spec.GetNoWrite() ? OpenExisting : Spec.GetCreateOnRead() ? OpenAlways : OpenExisting)); @@ -500,41 +506,42 @@ class TReplayRequestGeneratorFs final if (!fileHandle.IsOpen()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, - started, + Started, MakeError(E_FAIL, "fail")}); } const auto fh = fileHandle.GetHandle(); if (!fh) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, - started, + Started, MakeError(E_FAIL, "no filehandle")}); } OpenHandles[fh] = std::move(fileHandle); - HandlesLogToActual[r.GetNodeInfo().GetHandle()] = fh; + HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = fh; const auto inode = TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; - if (r.GetNodeInfo().GetNodeId()) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = inode; + if (logRequest.GetNodeInfo().GetNodeId()) { + NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = inode; NodePath[inode] = relativePathName; } STORAGE_DEBUG( - "open " << fh << "<-" << r.GetNodeInfo().GetHandle() + "open " << fh << "<-" << logRequest.GetNodeInfo().GetHandle() << " inode=" << inode << " known handles=" << HandlesLogToActual.size() << " opened=" << OpenHandles.size()); } catch (const TFileError& error) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, - started, + Started, MakeError(E_FAIL, error.what())}); } return MakeFuture( - TCompletedRequest{NProto::ACTION_CREATE_HANDLE, started, {}}); + TCompletedRequest{NProto::ACTION_CREATE_HANDLE, Started, {}}); } + static constexpr ui32 BlockSize = 4_KB; static std::shared_ptr Acalloc(ui64 dataSize) { @@ -548,56 +555,65 @@ class TReplayRequestGeneratorFs final return buffer; } + TFuture DoReadData( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { - const auto started = TInstant::Now(); if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_READ, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); + const auto handle = HandleIdMapped(logRequest.GetRanges(0).GetHandle()); if (!handle) { STORAGE_WARN( "read: no handle " - << r.GetRanges(0).GetHandle() - << " ranges size=" << r.GetRanges().size() + << logRequest.GetRanges(0).GetHandle() + << " ranges size=" << logRequest.GetRanges().size() << " map size=" << HandlesLogToActual.size()); return MakeFuture(TCompletedRequest{ NProto::ACTION_READ, - started, + Started, MakeError(E_FAIL, "cancelled")}); } auto& fh = OpenHandles[handle]; STORAGE_DEBUG( "Read from " << handle << " fh.len=" << fh.GetLength() << " fh.pos=" << fh.GetPosition()); - auto buffer = Acalloc(r.GetRanges().cbegin()->GetBytes()); + auto buffer = Acalloc(logRequest.GetRanges().cbegin()->GetBytes()); fh.Reserve( - r.GetRanges().cbegin()->GetOffset() + - r.GetRanges().cbegin()->GetBytes()); + logRequest.GetRanges().cbegin()->GetOffset() + + logRequest.GetRanges().cbegin()->GetBytes()); TFileHandle FileHandle{fh.GetHandle()}; const auto future = AsyncIO.Read( FileHandle, {}, - r.GetRanges().cbegin()->GetBytes(), - r.GetRanges().cbegin()->GetOffset()); + logRequest.GetRanges().cbegin()->GetBytes(), + logRequest.GetRanges().cbegin()->GetOffset()); FileHandle.Release(); return future.Apply( - [started]([[maybe_unused]] const auto& future) mutable - { return TCompletedRequest(NProto::ACTION_READ, started, {}); }); + [started = Started]([[maybe_unused]] const auto& future) mutable + { + if (future.GetValue()) { + return TCompletedRequest(NProto::ACTION_READ, started, {}); + } + return TCompletedRequest( + NProto::ACTION_READ, + started, + MakeError(E_IO, TStringBuilder{} << "nothing read")); + }); } - TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) + static TString + MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) { TStringBuilder ret; ret << "[\n" << start; @@ -608,32 +624,31 @@ class TReplayRequestGeneratorFs final } TFuture DoWrite( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} - const auto started = TInstant::Now(); if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_WRITE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - const auto logHandle = r.GetRanges(0).GetHandle(); + const auto logHandle = logRequest.GetRanges(0).GetHandle(); const auto handle = HandleIdMapped(logHandle); if (!handle) { return MakeFuture(TCompletedRequest( NProto::ACTION_WRITE, - started, + Started, MakeError( E_CANCELLED, TStringBuilder{} << "write cancelled: no handle =" << logHandle))); // todo } - const auto bytes = r.GetRanges(0).GetBytes(); - const auto offset = r.GetRanges(0).GetOffset(); + const auto bytes = logRequest.GetRanges(0).GetBytes(); + const auto offset = logRequest.GetRanges(0).GetOffset(); TString buffer; @@ -645,8 +660,8 @@ class TReplayRequestGeneratorFs final buffer = MakeBuffer( bytes, offset, - TStringBuilder{} << "handle=" << logHandle - << " node=" << r.GetNodeInfo().GetNodeId() + TStringBuilder{} << "handle=" << logHandle << " node=" + << logRequest.GetNodeInfo().GetNodeId() << " bytes=" << bytes << " offset=" << offset); } @@ -665,8 +680,16 @@ class TReplayRequestGeneratorFs final offset); FileHandle.Release(); return writeFuture.Apply( - [started]([[maybe_unused]] const auto& future) mutable - { return TCompletedRequest(NProto::ACTION_WRITE, started, {}); }); + [started = Started]([[maybe_unused]] const auto& future) mutable + { + if (future.GetValue()) { + return TCompletedRequest(NProto::ACTION_WRITE, started, {}); + } + return TCompletedRequest( + NProto::ACTION_WRITE, + started, + MakeError(E_IO, TStringBuilder{} << "nothing writed")); + }); } TString PathByNode(TNodeLocal nodeid) @@ -685,40 +708,40 @@ class TReplayRequestGeneratorFs final } TFuture DoCreateNode( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, // new_node_name=home, mode=509, node_id=12526, size=0} - const auto started = TInstant::Now(); if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - auto parentNode = NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + auto parentNode = + NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); - parentNode = - CreateDirIfMissingByNodeLog(r.GetNodeInfo().GetNewParentNodeId()); + parentNode = CreateDirIfMissingByNodeLog( + logRequest.GetNodeInfo().GetNewParentNodeId()); auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + - r.GetNodeInfo().GetNewNodeName(); + logRequest.GetNodeInfo().GetNewNodeName(); ui64 nodeid = 0; bool isDir = false; - switch (r.GetNodeInfo().GetType()) { + switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { // TODO: transform r.GetNodeInfo().GetMode() to correct open // mode TFileHandle fh( fullName, OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - fh.Reserve(r.GetNodeInfo().GetSize()); + fh.Reserve(logRequest.GetNodeInfo().GetSize()); if (fh) { nodeid = TFileStat{fh}.INode; } else { @@ -737,13 +760,13 @@ class TReplayRequestGeneratorFs final case NProto::E_SOCK_NODE: return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, - started, + Started, MakeError(E_NOT_IMPLEMENTED, "not implemented")}); case NProto::E_INVALID_NODE: return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, - started, + Started, MakeError(E_NOT_IMPLEMENTED, "not implemented")}); } @@ -753,44 +776,42 @@ class TReplayRequestGeneratorFs final // CreateIfMissing(PathByNode()) if (nodeid) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = nodeid; + NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = nodeid; NodePath[nodeid] = PathByNode(parentNode) + - r.GetNodeInfo().GetNewNodeName() + + logRequest.GetNodeInfo().GetNewNodeName() + (isDir ? "/" : ""); } return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); + TCompletedRequest(NProto::ACTION_CREATE_NODE, Started, {})); } TFuture DoRenameNode( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + { // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); - - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) - { - auto started = TInstant::Now(); if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_RENAME_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); const auto parentnodeid = - NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); auto fullName = Spec.GetReplayRoot() + PathByNode(parentnodeid) + - r.GetNodeInfo().GetNodeName(); + logRequest.GetNodeInfo().GetNodeName(); const auto newparentnodeid = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); auto newFullName = Spec.GetReplayRoot() + PathByNode(newparentnodeid) + - r.GetNodeInfo().GetNewNodeName(); + logRequest.GetNodeInfo().GetNewNodeName(); const auto renameres = NFs::Rename(fullName, newFullName); @@ -802,67 +823,66 @@ class TReplayRequestGeneratorFs final << LastSystemErrorText(LastSystemError())) : TStringBuilder{})); return MakeFuture( - TCompletedRequest{NProto::ACTION_RENAME_NODE, started, {}}); + TCompletedRequest{NProto::ACTION_RENAME_NODE, Started, {}}); } TFuture DoUnlinkNode( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} - auto started = TInstant::Now(); if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_REMOVE_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); const auto parentNodeId = - NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!parentNodeId) { STORAGE_WARN( "unlink : no parent orig=" - << r.GetNodeInfo().GetParentNodeId()); + << logRequest.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest( NProto::ACTION_REMOVE_NODE, - started, + Started, MakeError(E_CANCELLED, "cancelled"))); } const auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNodeId) + - r.GetNodeInfo().GetNodeName(); + logRequest.GetNodeInfo().GetNodeName(); const auto unlinkres = NFs::Remove(fullName); STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); // TODO : // NodesLogToActual.erase(...) // NodePath.erase(...) return MakeFuture( - TCompletedRequest(NProto::ACTION_REMOVE_NODE, started, {})); + TCompletedRequest(NProto::ACTION_REMOVE_NODE, Started, {})); } TFuture DoDestroyHandle( - NCloud::NFileStore::NProto::TProfileLogRequestInfo r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} TGuard guard(StateLock); - auto started = TInstant::Now(); - const auto handleid = HandleIdMapped(r.GetNodeInfo().GetHandle()); + const auto handleid = + HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); const auto& it = OpenHandles.find(handleid); if (it == OpenHandles.end()) { return MakeFuture(TCompletedRequest( NProto::ACTION_DESTROY_HANDLE, - started, + Started, MakeError( E_CANCELLED, TStringBuilder{} << "close " << handleid << " <- " - << r.GetNodeInfo().GetHandle() + << logRequest.GetNodeInfo().GetHandle() << " fail: not found in " << OpenHandles.size()))); } @@ -872,24 +892,24 @@ class TReplayRequestGeneratorFs final const auto pos = fhandle.GetPosition(); fhandle.Close(); OpenHandles.erase(handleid); - HandlesLogToActual.erase(r.GetNodeInfo().GetHandle()); + HandlesLogToActual.erase(logRequest.GetNodeInfo().GetHandle()); STORAGE_DEBUG( - "Close " << handleid << " <- " << r.GetNodeInfo().GetHandle() - << " pos=" << pos << " len=" << len + "Close " << handleid << " <- " + << logRequest.GetNodeInfo().GetHandle() << " pos=" << pos + << " len=" << len << " open map size=" << OpenHandles.size() << " map size=" << HandlesLogToActual.size()); return MakeFuture( - TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, started, {})); + TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, Started, {})); } TFuture DoGetNodeAttr( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { - const auto started = TInstant::Now(); if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_GET_NODE_ATTR, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } @@ -902,32 +922,33 @@ class TReplayRequestGeneratorFs final // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} - if (r.GetNodeInfo().GetNodeName()) { - KnownLogNodes[r.GetNodeInfo().GetNodeId()].Name = - r.GetNodeInfo().GetNodeName(); + if (logRequest.GetNodeInfo().GetNodeName()) { + KnownLogNodes[logRequest.GetNodeInfo().GetNodeId()].Name = + logRequest.GetNodeInfo().GetNodeName(); } - if (r.GetNodeInfo().GetParentNodeId() && - r.GetNodeInfo().GetParentNodeId() != r.GetNodeInfo().GetNodeId()) + if (logRequest.GetNodeInfo().GetParentNodeId() && + logRequest.GetNodeInfo().GetParentNodeId() != + logRequest.GetNodeInfo().GetNodeId()) { - KnownLogNodes[r.GetNodeInfo().GetNodeId()].ParentLog = - r.GetNodeInfo().GetParentNodeId(); + KnownLogNodes[logRequest.GetNodeInfo().GetNodeId()].ParentLog = + logRequest.GetNodeInfo().GetParentNodeId(); } // TODO: can create and truncate to size here missing files - const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + const auto nodeid = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); if (!nodeid) { return MakeFuture(TCompletedRequest{ NProto::ACTION_GET_NODE_ATTR, - started, + Started, MakeError(E_CANCELLED, "cancelled")}); } auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); [[maybe_unused]] const auto stat = TFileStat{fname}; return MakeFuture( - TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, started, {})); + TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, Started, {})); } TFuture DoAcquireLock() @@ -937,7 +958,6 @@ class TReplayRequestGeneratorFs final // return DoCreateHandle(); } - auto started = TInstant::Now(); auto it = Handles.begin(); while (it != Handles.end() && (Locks.contains(it->first) || StagedLocks.contains(it->first))) @@ -948,7 +968,7 @@ class TReplayRequestGeneratorFs final if (it == Handles.end()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_ACQUIRE_LOCK, - started, + Started, MakeError(E_CANCELLED, "cancelled")}); } @@ -963,7 +983,8 @@ class TReplayRequestGeneratorFs final auto self = weak_from_this(); return Session->AcquireLock(CreateCallContext(), std::move(request)) .Apply( - [=](const TFuture& future) + [=, started = Started]( + const TFuture& future) { if (auto ptr = self.lock()) { return ptr->HandleAcquireLock(handle, future, started); @@ -1027,11 +1048,11 @@ class TReplayRequestGeneratorFs final request->SetOwner(OwnerId); request->SetLength(LockLength); - auto started = TInstant::Now(); auto self = weak_from_this(); return Session->ReleaseLock(CreateCallContext(), std::move(request)) .Apply( - [=](const TFuture& future) + [=, started = Started]( + const TFuture& future) { if (auto ptr = self.lock()) { return ptr->HandleReleaseLock(handle, future, started); @@ -1071,18 +1092,17 @@ class TReplayRequestGeneratorFs final } TFuture DoListNodes( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // json={"TimestampMcs":1726615510721016,"DurationMcs":3329,"RequestType":30,"ErrorCode":0,"NodeInfo":{"NodeId":164,"Size":10}} - auto started = TInstant::Now(); TGuard guard(StateLock); - const auto nodeid = NodeIdMapped(r.GetNodeInfo().GetNodeId()); + const auto nodeid = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); if (!nodeid) { return MakeFuture(TCompletedRequest{ NProto::ACTION_LIST_NODES, - started, + Started, MakeError( E_CANCELLED, TStringBuilder{} << "Node not found in mapping" @@ -1098,19 +1118,19 @@ class TReplayRequestGeneratorFs final if (!dir.IsOpen()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_LIST_NODES, - started, + Started, MakeError(E_CANCELLED, "cancelled")}); } const auto dirs = NLowLevel::ListDirAt(dir, true); - if (r.GetNodeInfo().GetSize() != dirs.size()) { + if (logRequest.GetNodeInfo().GetSize() != dirs.size()) { STORAGE_DEBUG( - "Dir size differs " << path - << " log=" << r.GetNodeInfo().GetSize() - << " local=" << dirs.size()); + "Dir size differs " + << path << " log=" << logRequest.GetNodeInfo().GetSize() + << " local=" << dirs.size()); } return MakeFuture( - TCompletedRequest(NProto::ACTION_LIST_NODES, started, {})); + TCompletedRequest(NProto::ACTION_LIST_NODES, Started, {})); } }; diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 677fa9e8e8..a9157c460d 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -13,8 +13,6 @@ compare log and actual result ( S_OK E_FS_NOENT ...) #include "cloud/filestore/tools/analytics/libs/event-log/dump.h" #include "library/cpp/eventlog/iterator.h" #include "library/cpp/testing/unittest/registar.h" -#include "util/folder/dirut.h" -#include "util/folder/path.h" #include "util/system/fs.h" #include @@ -203,44 +201,47 @@ class TReplayRequestGeneratorGRPC final NThreading::TFuture ExecuteNextRequest() override { if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest{}); + return MakeFuture(TCompletedRequest(true)); } + for (; EventPtr; Advance()) { if (!EventLogMessagePtr) { continue; } - STORAGE_DEBUG("Processing %d", EventMessageNumber); for (; EventMessageNumber > 0;) { auto request = EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - STORAGE_DEBUG("message json=" << request.AsJSON()); - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - TimestampMcs = request.GetTimestampMcs(); - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - if (timediff > diff.MicroSeconds()) { - auto slp = - TDuration::MicroSeconds(timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); + { + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = TDuration::MicroSeconds( + timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); - Sleep(slp); + Sleep(slp); + } + Started = current; } - Started = current; STORAGE_DEBUG( "Processing message " << EventMessageNumber << " typename=" << request.GetTypeName() << " type=" << request.GetRequestType() << " " - << RequestName(request.GetRequestType())); + << " name=" << RequestName(request.GetRequestType()) + << " json=" << request.AsJSON()); { const auto& action = request.GetRequestType(); @@ -294,16 +295,14 @@ class TReplayRequestGeneratorGRPC final private: TFuture DoAccessNode( [[maybe_unused]] const NCloud::NFileStore::NProto:: - TProfileLogRequestInfo& r) + TProfileLogRequestInfo& logRequest) { // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} - auto started = TInstant::Now(); - if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } @@ -311,12 +310,12 @@ class TReplayRequestGeneratorGRPC final return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, - started, + Started, MakeError(E_NOT_IMPLEMENTED, "not implemented")}); } TFuture DoCreateHandle( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} @@ -328,16 +327,17 @@ class TReplayRequestGeneratorGRPC final auto started = TInstant::Now(); auto request = CreateRequest(); - auto name = r.GetNodeInfo().GetNodeName(); + auto name = logRequest.GetNodeInfo().GetNodeName(); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + const auto node = + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{}); } request->SetNodeId(node); - request->SetName(r.GetNodeInfo().GetNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); - request->SetMode(r.GetNodeInfo().GetMode()); + request->SetName(logRequest.GetNodeInfo().GetNodeName()); + request->SetFlags(logRequest.GetNodeInfo().GetFlags()); + request->SetMode(logRequest.GetNodeInfo().GetMode()); auto self = weak_from_this(); return Session->CreateHandle(CreateCallContext(), std::move(request)) @@ -346,8 +346,11 @@ class TReplayRequestGeneratorGRPC final const TFuture& future) { if (auto ptr = self.lock()) { - return ptr - ->HandleCreateHandle(future, name, started, r); + return ptr->HandleCreateHandle( + future, + name, + started, + logRequest); } return MakeFuture(TCompletedRequest{ @@ -361,7 +364,7 @@ class TReplayRequestGeneratorGRPC final const TFuture& future, const TString& name, TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { try { auto response = future.GetValue(); @@ -369,7 +372,8 @@ class TReplayRequestGeneratorGRPC final auto handle = response.GetHandle(); with_lock (StateLock) { - HandlesLogToActual[r.GetNodeInfo().GetHandle()] = handle; + HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = + handle; } NThreading::TFuture setAttr; @@ -426,7 +430,7 @@ class TReplayRequestGeneratorGRPC final return buffer; } TFuture DoReadData( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { const auto started = TInstant::Now(); if (Spec.GetNoRead()) { @@ -439,13 +443,14 @@ class TReplayRequestGeneratorGRPC final TGuard guard(StateLock); auto request = CreateRequest(); - const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + const auto handle = + HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); if (!handle) { return MakeFuture(TCompletedRequest{}); } request->SetHandle(handle); - request->SetOffset(r.GetRanges().cbegin()->GetOffset()); - request->SetLength(r.GetRanges().cbegin()->GetBytes()); + request->SetOffset(logRequest.GetRanges().cbegin()->GetOffset()); + request->SetLength(logRequest.GetRanges().cbegin()->GetBytes()); auto self = weak_from_this(); return Session->ReadData(CreateCallContext(), std::move(request)) @@ -502,7 +507,7 @@ class TReplayRequestGeneratorGRPC final } TFuture DoWrite( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} @@ -516,15 +521,15 @@ class TReplayRequestGeneratorGRPC final TGuard guard(StateLock); auto request = CreateRequest(); - const auto handle = HandleIdMapped(r.GetRanges(0).GetHandle()); + const auto handle = HandleIdMapped(logRequest.GetRanges(0).GetHandle()); if (!handle) { return MakeFuture(TCompletedRequest{}); } request->SetHandle(handle); - request->SetOffset(r.GetRanges(0).GetOffset()); - auto bytes = r.GetRanges(0).GetBytes(); + request->SetOffset(logRequest.GetRanges(0).GetOffset()); + auto bytes = logRequest.GetRanges(0).GetBytes(); auto buffer = NUnitTest::RandomString(bytes); *request->MutableBuffer() = std::move(buffer); @@ -582,17 +587,16 @@ class TReplayRequestGeneratorGRPC final } TFuture DoCreateNode( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, // new_node_name=home, mode=509, node_id=12526, size=0} - const auto started = TInstant::Now(); if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } @@ -601,23 +605,25 @@ class TReplayRequestGeneratorGRPC final auto request = CreateRequest(); const auto parentNode = - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId()); + NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); if (!parentNode) { return MakeFuture(TCompletedRequest{}); } request->SetNodeId(parentNode); - auto name = r.GetNodeInfo().GetNewNodeName(); - request->SetName(r.GetNodeInfo().GetNewNodeName()); + auto name = logRequest.GetNodeInfo().GetNewNodeName(); + request->SetName(logRequest.GetNodeInfo().GetNewNodeName()); // request->SetGid(); // request->SetUid(); - switch (r.GetNodeInfo().GetType()) { + switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: - request->MutableFile()->SetMode(r.GetNodeInfo().GetMode()); + request->MutableFile()->SetMode( + logRequest.GetNodeInfo().GetMode()); break; case NProto::E_DIRECTORY_NODE: - request->MutableDirectory()->SetMode(r.GetNodeInfo().GetMode()); + request->MutableDirectory()->SetMode( + logRequest.GetNodeInfo().GetMode()); break; case NProto::E_LINK_NODE: // TODO: @@ -630,11 +636,12 @@ class TReplayRequestGeneratorGRPC final // request->MutableSymlink()->SetTargetPath(); break; case NProto::E_SOCK_NODE: - request->MutableSocket()->SetMode(r.GetNodeInfo().GetMode()); + request->MutableSocket()->SetMode( + logRequest.GetNodeInfo().GetMode()); break; case NProto::E_INVALID_NODE: return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_NODE, started, {})); + TCompletedRequest(NProto::ACTION_CREATE_NODE, Started, {})); // Do not create files with invalid // type - too hard to delete them break; @@ -643,10 +650,15 @@ class TReplayRequestGeneratorGRPC final auto self = weak_from_this(); return Session->CreateNode(CreateCallContext(), std::move(request)) .Apply( - [=](const TFuture& future) + [=, started = Started]( + const TFuture& future) { if (auto ptr = self.lock()) { - return ptr->HandleCreateNode(future, name, started, r); + return ptr->HandleCreateNode( + future, + name, + started, + logRequest); } return TCompletedRequest{ @@ -660,7 +672,7 @@ class TReplayRequestGeneratorGRPC final const TFuture& future, const TString& name, TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { TGuard guard(StateLock); @@ -669,7 +681,7 @@ class TReplayRequestGeneratorGRPC final CheckResponse(response); // Nodes[name] = TNode{name, response.GetNode()}; if (response.GetNode().GetId()) { - NodesLogToLocal[r.GetNodeInfo().GetNodeId()] = + NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = response.GetNode().GetId(); } @@ -686,51 +698,53 @@ class TReplayRequestGeneratorGRPC final } TFuture DoRenameNode( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + { // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); - - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) - { - auto started = TInstant::Now(); if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_RENAME_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); auto request = CreateRequest(); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + const auto node = + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{ NProto::ACTION_RENAME_NODE, - started, + Started, MakeError( E_FAIL, - TStringBuilder{} << "Log node " - << r.GetNodeInfo().GetParentNodeId() - << "not found in mappiong")}); + TStringBuilder{} + << "Log node " + << logRequest.GetNodeInfo().GetParentNodeId() + << "not found in mappiong")}); } request->SetNodeId(node); - request->SetName(r.GetNodeInfo().GetNodeName()); + request->SetName(logRequest.GetNodeInfo().GetNodeName()); request->SetNewParentId( - NodeIdMapped(r.GetNodeInfo().GetNewParentNodeId())); - request->SetNewName(r.GetNodeInfo().GetNewNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); + NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId())); + request->SetNewName(logRequest.GetNodeInfo().GetNewNodeName()); + request->SetFlags(logRequest.GetNodeInfo().GetFlags()); auto self = weak_from_this(); return Session->RenameNode(CreateCallContext(), std::move(request)) .Apply( - [= - - ](const TFuture& future) + [=, started = Started]( + const TFuture& future) { if (auto ptr = self.lock()) { - return ptr->HandleRenameNode(future, started, r); + return ptr->HandleRenameNode( + future, + started, + logRequest); } return TCompletedRequest{ @@ -743,7 +757,7 @@ class TReplayRequestGeneratorGRPC final TCompletedRequest HandleRenameNode( const TFuture& future, TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { TGuard guard(StateLock); try { @@ -754,32 +768,32 @@ class TReplayRequestGeneratorGRPC final auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); STORAGE_ERROR( "rename node %s has failed: %s", - r.GetNodeInfo().GetNodeName().c_str(), + logRequest.GetNodeInfo().GetNodeName().c_str(), FormatError(error).c_str()); return {NProto::ACTION_RENAME_NODE, started, error}; } } TFuture DoUnlinkNode( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} - auto started = TInstant::Now(); if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_REMOVE_NODE, - started, + Started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } TGuard guard(StateLock); - auto name = r.GetNodeInfo().GetNodeName(); + auto name = logRequest.GetNodeInfo().GetNodeName(); auto request = CreateRequest(); request->SetName(name); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + const auto node = + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{}); } @@ -787,7 +801,7 @@ class TReplayRequestGeneratorGRPC final auto self = weak_from_this(); return Session->UnlinkNode(CreateCallContext(), std::move(request)) .Apply( - [=, name = std::move(name)]( + [=, started = Started, name = std::move(name)]( const TFuture& future) { if (auto ptr = self.lock()) { @@ -822,18 +836,18 @@ class TReplayRequestGeneratorGRPC final } } TFuture DoDestroyHandle( - NCloud::NFileStore::NProto::TProfileLogRequestInfo r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} TGuard guard(StateLock); - auto started = TInstant::Now(); - auto name = r.GetNodeInfo().GetNodeName(); + auto name = logRequest.GetNodeInfo().GetNodeName(); auto request = CreateRequest(); - const auto handle = HandleIdMapped(r.GetNodeInfo().GetHandle()); + const auto handle = + HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); if (!handle) { return MakeFuture(TCompletedRequest{}); } @@ -845,7 +859,7 @@ class TReplayRequestGeneratorGRPC final auto self = weak_from_this(); return Session->DestroyHandle(CreateCallContext(), std::move(request)) .Apply( - [=, name = std::move(name)]( + [=, started = Started, name = std::move(name)]( const TFuture& future) { if (auto ptr = self.lock()) { @@ -860,7 +874,7 @@ class TReplayRequestGeneratorGRPC final } TFuture DoGetNodeAttr( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& r) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { const auto started = TInstant::Now(); if (Spec.GetNoRead()) { @@ -880,14 +894,15 @@ class TReplayRequestGeneratorGRPC final // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} auto request = CreateRequest(); - const auto node = NodeIdMapped(r.GetNodeInfo().GetParentNodeId()); + const auto node = + NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!node) { return MakeFuture(TCompletedRequest{}); } request->SetNodeId(node); - auto name = r.GetNodeInfo().GetNodeName(); - request->SetName(r.GetNodeInfo().GetNodeName()); - request->SetFlags(r.GetNodeInfo().GetFlags()); + auto name = logRequest.GetNodeInfo().GetNodeName(); + request->SetName(logRequest.GetNodeInfo().GetNodeName()); + request->SetFlags(logRequest.GetNodeInfo().GetFlags()); auto self = weak_from_this(); STORAGE_DEBUG("GetNodeAttr client started"); return Session->GetNodeAttr(CreateCallContext(), std::move(request)) From 823e1b70e629a6e83fde31256b9a46341d4b9efb Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 26 Sep 2024 01:00:43 +0000 Subject: [PATCH 12/44] grpc fix --- .../loadtest/lib/request_replay_grpc.cpp | 139 ++++++++++++------ .../tools/testing/loadtest/lib/test.cpp | 11 +- 2 files changed, 102 insertions(+), 48 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index a9157c460d..727cce0ff0 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -140,6 +140,8 @@ class TReplayRequestGeneratorGRPC final TNodeLocal NodeIdMapped(const TNodeLog id) { + TGuard guard(StateLock); + if (const auto it = NodesLogToLocal.find(id); it != NodesLogToLocal.end()) { @@ -153,6 +155,8 @@ class TReplayRequestGeneratorGRPC final THandleLocal HandleIdMapped(const THandleLog id) { + TGuard guard(StateLock); + if (const auto it = HandlesLogToActual.find(id); it != HandlesLogToActual.end()) { @@ -323,9 +327,6 @@ class TReplayRequestGeneratorGRPC final // node_name=ini, flags=14, mode=436, node_id=66, // handle=11024287581389312, size=0} - TGuard guard(StateLock); - auto started = TInstant::Now(); - auto request = CreateRequest(); auto name = logRequest.GetNodeInfo().GetNodeName(); @@ -339,25 +340,39 @@ class TReplayRequestGeneratorGRPC final request->SetFlags(logRequest.GetNodeInfo().GetFlags()); request->SetMode(logRequest.GetNodeInfo().GetMode()); + STORAGE_DEBUG( + "open " << " handle=" << logRequest.GetNodeInfo().GetHandle() + << " flags=" << logRequest.GetNodeInfo().GetFlags() << " " + << HandleFlagsToString(logRequest.GetNodeInfo().GetFlags()) + << " mode=" << logRequest.GetNodeInfo().GetMode() + << " node=" << node << " <- " + << logRequest.GetNodeInfo().GetNodeId()); + auto self = weak_from_this(); - return Session->CreateHandle(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateHandle( - future, - name, + const auto future = + Session->CreateHandle(CreateCallContext(), std::move(request)) + .Apply( + [=, started = Started, name = std::move(name)]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleCreateHandle( + future, + name, + started, + logRequest); + } + + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, started, - logRequest); - } - - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "cancelled")}); - }); + MakeError(E_FAIL, "cancelled")}); + }); + const auto& response = future.GetValueSync(); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_HANDLE, + Started, + response.Error}); } TFuture HandleCreateHandle( @@ -518,19 +533,36 @@ class TReplayRequestGeneratorGRPC final started, MakeError(E_PRECONDITION_FAILED, "disabled")}); } + TGuard guard(StateLock); auto request = CreateRequest(); - const auto handle = HandleIdMapped(logRequest.GetRanges(0).GetHandle()); + const auto logHandle = logRequest.GetRanges(0).GetHandle(); + const auto handle = HandleIdMapped(logHandle); if (!handle) { return MakeFuture(TCompletedRequest{}); } request->SetHandle(handle); - request->SetOffset(logRequest.GetRanges(0).GetOffset()); + const auto offset = logRequest.GetRanges(0).GetOffset(); + request->SetOffset(offset); auto bytes = logRequest.GetRanges(0).GetBytes(); - auto buffer = NUnitTest::RandomString(bytes); + + TString buffer; + + if (Spec.GetWriteRandom()) { + buffer = NUnitTest::RandomString(bytes, logHandle); + } else if (Spec.GetWriteEmpty()) { + buffer = TString{bytes, ' '}; + } else { + buffer = MakeBuffer( + bytes, + offset, + TStringBuilder{} << "handle=" << logHandle << " node=" + << logRequest.GetNodeInfo().GetNodeId() + << " bytes=" << bytes << " offset=" << offset); + } *request->MutableBuffer() = std::move(buffer); @@ -600,8 +632,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - auto request = CreateRequest(); const auto parentNode = @@ -648,24 +678,30 @@ class TReplayRequestGeneratorGRPC final } Cerr << "createnoderec" << *request << "\n"; auto self = weak_from_this(); - return Session->CreateNode(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateNode( - future, - name, + const auto future = + Session->CreateNode(CreateCallContext(), std::move(request)) + .Apply( + [=, started = Started]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleCreateNode( + future, + name, + started, + logRequest); + } + + return TCompletedRequest{ + NProto::ACTION_CREATE_NODE, started, - logRequest); - } - - return TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + MakeError(E_CANCELLED, "cancelled")}; + }); + const auto& response = future.GetValueSync(); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_CREATE_NODE, + Started, + response.Error}); } TCompletedRequest HandleCreateNode( @@ -674,9 +710,8 @@ class TReplayRequestGeneratorGRPC final TInstant started, const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { - TGuard guard(StateLock); - try { + TGuard guard(StateLock); auto response = future.GetValue(); CheckResponse(response); // Nodes[name] = TNode{name, response.GetNode()}; @@ -887,7 +922,6 @@ class TReplayRequestGeneratorGRPC final TGuard guard(StateLock); // TODO: by parent + name // - // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, @@ -897,14 +931,25 @@ class TReplayRequestGeneratorGRPC final const auto node = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!node) { - return MakeFuture(TCompletedRequest{}); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_GET_NODE_ATTR, + started, + MakeError( + E_NOT_FOUND, + TStringBuilder{} + << "Node missing in mapping " + << logRequest.GetNodeInfo().GetParentNodeId())}); } request->SetNodeId(node); auto name = logRequest.GetNodeInfo().GetNodeName(); request->SetName(logRequest.GetNodeInfo().GetNodeName()); request->SetFlags(logRequest.GetNodeInfo().GetFlags()); auto self = weak_from_this(); - STORAGE_DEBUG("GetNodeAttr client started"); + STORAGE_DEBUG( + "GetNodeAttr client started name=" + << name << " node=" << node << " <- " + << logRequest.GetNodeInfo().GetParentNodeId() // why only parent? + << " request=" << *request); return Session->GetNodeAttr(CreateCallContext(), std::move(request)) .Apply( [=, name = std::move(name)]( diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 3242f744b1..32fcd7f7e4 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -601,7 +601,16 @@ class TLoadTest final } }); if (RequestGenerator->InstantProcessQueue()) { - if (future.HasValue()) { + if (future.HasException()) { + try { + future.TryRethrow(); + } catch (const std::exception& ex) { + DUMP(ex.what()); + } + --CurrentIoDepth; + } + + if (future.HasValue() || future.HasException()) { ProcessCompletedRequests(); } } From e3da72904978f66cfe50e0961a41cd508bfc7c97 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 27 Sep 2024 16:10:21 +0000 Subject: [PATCH 13/44] wip --- cloud/filestore/libs/client/session.cpp | 2 +- .../loadtest/lib/request_replay_grpc.cpp | 78 ++++++++++--------- .../tools/testing/loadtest/lib/test.cpp | 18 ++--- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cloud/filestore/libs/client/session.cpp b/cloud/filestore/libs/client/session.cpp index 1ce66a0a7a..e1198939a6 100644 --- a/cloud/filestore/libs/client/session.cpp +++ b/cloud/filestore/libs/client/session.cpp @@ -461,7 +461,7 @@ FILESTORE_SESSION_FORWARD(FILESTORE_IMPLEMENT_METHOD) with_lock (SessionLock) { if (!HasError(response)) { STORAGE_INFO(LogTag(GetSessionId(response), GetSessionSeqNo(response)) - << " session established" << GetSessionState(response).size()); + << " session established " << GetSessionState(response).size()); SessionState = SessionEstablished; diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 727cce0ff0..3a537c4fb5 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -310,8 +310,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, Started, @@ -455,8 +453,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - auto request = CreateRequest(); const auto handle = HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); @@ -534,8 +530,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - auto request = CreateRequest(); const auto logHandle = logRequest.GetRanges(0).GetHandle(); const auto handle = HandleIdMapped(logHandle); @@ -711,13 +705,14 @@ class TReplayRequestGeneratorGRPC final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { try { - TGuard guard(StateLock); auto response = future.GetValue(); CheckResponse(response); // Nodes[name] = TNode{name, response.GetNode()}; if (response.GetNode().GetId()) { - NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = - response.GetNode().GetId(); + with_lock (StateLock) { + NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = + response.GetNode().GetId(); + } } return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; @@ -746,8 +741,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - auto request = CreateRequest(); const auto node = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); @@ -770,23 +763,29 @@ class TReplayRequestGeneratorGRPC final request->SetFlags(logRequest.GetNodeInfo().GetFlags()); auto self = weak_from_this(); - return Session->RenameNode(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleRenameNode( - future, - started, - logRequest); - } + const auto future = + Session->RenameNode(CreateCallContext(), std::move(request)) + .Apply( + [=, started = Started]( + const TFuture& future) + { + if (auto ptr = self.lock()) { + return ptr->HandleRenameNode( + future, + started, + logRequest); + } - return TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); + return TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + started, + MakeError(E_INVALID_STATE, "invalid ptr")}; + }); + const auto& response = future.GetValueSync(); + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RENAME_NODE, + Started, + response.Error}); } TCompletedRequest HandleRenameNode( @@ -794,7 +793,6 @@ class TReplayRequestGeneratorGRPC final TInstant started, const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { - TGuard guard(StateLock); try { auto response = future.GetValue(); CheckResponse(response); @@ -822,8 +820,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - auto name = logRequest.GetNodeInfo().GetNodeName(); auto request = CreateRequest(); request->SetName(name); @@ -875,7 +871,6 @@ class TReplayRequestGeneratorGRPC final { // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} - TGuard guard(StateLock); auto name = logRequest.GetNodeInfo().GetNodeName(); @@ -886,8 +881,9 @@ class TReplayRequestGeneratorGRPC final if (!handle) { return MakeFuture(TCompletedRequest{}); } - - HandlesLogToActual.erase(handle); + with_lock (StateLock) { + HandlesLogToActual.erase(handle); + } request->SetHandle(handle); @@ -919,8 +915,6 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - TGuard guard(StateLock); - // TODO: by parent + name // // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} @@ -975,7 +969,6 @@ class TReplayRequestGeneratorGRPC final auto response = future.GetValue(); STORAGE_DEBUG("GetNodeAttr client completed"); CheckResponse(response); - TGuard guard(StateLock); return {NProto::ACTION_GET_NODE_ATTR, started, {}}; } catch (const TServiceError& e) { auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); @@ -984,7 +977,7 @@ class TReplayRequestGeneratorGRPC final name.c_str(), FormatError(error).c_str()); - return {NProto::ACTION_GET_NODE_ATTR, started, {}}; + return {NProto::ACTION_GET_NODE_ATTR, started, error}; } } @@ -1011,6 +1004,12 @@ class TReplayRequestGeneratorGRPC final TFuture DoAcquireLock() { + // TODO: + return MakeFuture(TCompletedRequest{ + NProto::ACTION_ACQUIRE_LOCK, + Started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + TGuard guard(StateLock); if (Handles.empty()) { // return DoCreateHandle(); @@ -1090,6 +1089,11 @@ class TReplayRequestGeneratorGRPC final TFuture DoReleaseLock() { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + Started, + MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + TGuard guard(StateLock); if (Locks.empty()) { return DoAcquireLock(); diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 32fcd7f7e4..7c85776a17 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -597,19 +597,14 @@ class TLoadTest final [=](const TFuture& future) { if (auto ptr = self.lock()) { - ptr->SignalCompletion(future.GetValue()); + if (future.HasException()) { + ptr->SignalCompletion(TCompletedRequest{}); + } else { + ptr->SignalCompletion(future.GetValue()); + } } }); if (RequestGenerator->InstantProcessQueue()) { - if (future.HasException()) { - try { - future.TryRethrow(); - } catch (const std::exception& ex) { - DUMP(ex.what()); - } - --CurrentIoDepth; - } - if (future.HasValue() || future.HasException()) { ProcessCompletedRequests(); } @@ -635,8 +630,9 @@ class TLoadTest final auto code = request->Error.GetCode(); if (FAILED(code)) { - STORAGE_ERROR("%s failing test due to: %s", + STORAGE_ERROR("%s failing test %d due to: %s", MakeTestTag().c_str(), + request->Action, FormatError(request->Error).c_str()); if (RequestGenerator->FailOnError()) { From a0c0eff723d68a5668f2a5bcfd1af77e8176d180 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 27 Sep 2024 19:32:57 +0000 Subject: [PATCH 14/44] fix resize --- .../loadtest/lib/request_replay_fs.cpp | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 292f660b5c..109248430c 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -586,9 +586,13 @@ class TReplayRequestGeneratorFs final << " fh.pos=" << fh.GetPosition()); auto buffer = Acalloc(logRequest.GetRanges().cbegin()->GetBytes()); - fh.Reserve( - logRequest.GetRanges().cbegin()->GetOffset() + - logRequest.GetRanges().cbegin()->GetBytes()); + /* + if (!fh.GetLength()) { + // incorrect aligned to read size, should use size from + nodeattr fh.Reserve( logRequest.GetRanges().cbegin()->GetOffset() + + logRequest.GetRanges().cbegin()->GetBytes()); + } + */ TFileHandle FileHandle{fh.GetHandle()}; @@ -741,12 +745,15 @@ class TReplayRequestGeneratorFs final TFileHandle fh( fullName, OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); - fh.Reserve(logRequest.GetNodeInfo().GetSize()); if (fh) { nodeid = TFileStat{fh}.INode; } else { nodeid = TFileStat{fullName}.INode; } + + if (logRequest.GetNodeInfo().GetSize()) { + fh.Reserve(logRequest.GetNodeInfo().GetSize()); + } } break; case NProto::E_DIRECTORY_NODE: { isDir = true; @@ -915,10 +922,11 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); - // TODO: by parent + name // + // TODO: by ParentNodeId + NodeName // // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} + // {"TimestampMcs":1727464381415468,"DurationMcs":1982,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":2,"NodeName":"libc.so.6","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} @@ -942,7 +950,11 @@ class TReplayRequestGeneratorFs final return MakeFuture(TCompletedRequest{ NProto::ACTION_GET_NODE_ATTR, Started, - MakeError(E_CANCELLED, "cancelled")}); + MakeError( + E_NOT_FOUND, + TStringBuilder{} << "Node not found " + << logRequest.GetNodeInfo().GetNodeId() + << " in " << NodesLogToLocal.size())}); } auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); From d3d1f67583f6f6b29e39e36301ad66b6878a5e85 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 27 Sep 2024 19:44:26 +0000 Subject: [PATCH 15/44] Test scripts --- cloud/filestore/bin/initctl.sh | 26 ++++-- .../tools/testing/loadtest/test/lib.sh | 83 +++++++++++++++++++ .../tools/testing/loadtest/test/load1.sh | 3 + .../tools/testing/loadtest/test/load3.sh | 6 ++ .../tools/testing/loadtest/test/record.sh | 35 ++++++++ .../tools/testing/loadtest/test/replay_fs.sh | 42 ++++++++++ .../testing/loadtest/test/replay_fs.txt.tmpl | 18 ++++ .../testing/loadtest/test/replay_grpc.sh | 37 +++++++++ .../loadtest/test/replay_grpc.txt.tmpl | 17 ++++ 9 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 cloud/filestore/tools/testing/loadtest/test/lib.sh create mode 100755 cloud/filestore/tools/testing/loadtest/test/load1.sh create mode 100755 cloud/filestore/tools/testing/loadtest/test/load3.sh create mode 100755 cloud/filestore/tools/testing/loadtest/test/record.sh create mode 100755 cloud/filestore/tools/testing/loadtest/test/replay_fs.sh create mode 100644 cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl create mode 100755 cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh create mode 100644 cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl diff --git a/cloud/filestore/bin/initctl.sh b/cloud/filestore/bin/initctl.sh index c1f50068ae..f839f7401d 100755 --- a/cloud/filestore/bin/initctl.sh +++ b/cloud/filestore/bin/initctl.sh @@ -7,18 +7,26 @@ find_bin_dir() { } BIN_DIR=`find_bin_dir` -LOGS_DIR=$BIN_DIR/data +export LOGS_DIR=$BIN_DIR/data -SERVER_PORT=${SERVER_PORT:-9021} -VHOST_PORT=${VHOST_PORT:-9022} +export SERVER_PORT=${SERVER_PORT:-9021} +export VHOST_PORT=${VHOST_PORT:-9022} -FS=${FS:-"nfs"} -SHARD_COUNT=${SHARD_COUNT:-0} -BLOCK_SIZE=${BLOCK_SIZE:-4096} -MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"} -VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock} +export FS=${FS:-"nfs"} +export SHARD_COUNT=${SHARD_COUNT:-0} +export BLOCK_SIZE=${BLOCK_SIZE:-4096} +export MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"} +export VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock} -PID_FILE=$BIN_DIR/pids.txt +export PID_FILE=$BIN_DIR/pids.txt + +if [ -z "$FILESTORE_APPS_PATH" ]; then + REPO_ROOT=$BIN_DIR/../../.. + FILESTORE_APPS_PATH=cloud/filestore/buildall/cloud/filestore/apps + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/client + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/server + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/vhost +fi ################################################################################ # STOP/KILL diff --git a/cloud/filestore/tools/testing/loadtest/test/lib.sh b/cloud/filestore/tools/testing/loadtest/test/lib.sh new file mode 100644 index 0000000000..a1f92e53a9 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/lib.sh @@ -0,0 +1,83 @@ + +if [ -z ${CONFIGURED} ]; then +export CONFIGURED=1 + +[ -z "${VERBOSE=${V}}" ] && set +x +[ -n "${VERBOSE=${V}}" ] && set -x +[ -z "${KEEP}" ] && set -e + +#export ARCADIA_ROOT=${ARCADIA_ROOT:=`ya dump root || echo ~/arcadia/`} + +ROOT_DIR=$(git rev-parse --show-toplevel) + + +# CUR_DIR=${CUR_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} + +# YA_MAKE_ADD=--sanitize=address +# YA_MAKE_ADD=--sanitize=memory +# YA_MAKE_ADD=--sanitize=thread +# YA_MAKE_ADD=--sanitize=undefined +# YA_MAKE_ADD=--sanitize=leak +# YA_MAKE_ADD=-r + +#YA_MAKE_ADD=${YA_MAKE_ADD=-r} +#YA_MAKE=${YA_MAKE="ya make ${YA_MAKE_ADD}"} # " + +REBUILD=${REBUILD=1} + +# STARTER='ya tool gdb -ex run --args' +# STARTER='ya tool gdb -ex run -ex backtrace --args' +# STARTER='GRPC_VERBOSITY=info GRPC_TRACE=tcp,http,api' + + + + +function re_create_mount { +pushd ${ROOT_DIR} +ya make -r cloud/filestore/buildall +popd + +pushd ${ROOT_DIR}/cloud/filestore/bin + +. initctl.sh +umount ${MOUNT_POINT} ||: +. initctl.sh stop + +. initctl.sh format initialize +. initctl.sh create startendpoint +. initctl.sh start +. initctl.sh mount + +sleep 3 +echo mountpoint = ${MOUNT_POINT} +popd + +} + + +# === Library section + +# Add pid of last runned program to "kill on exit" list +function last_killer { + [ -n "${NO_KILL}" ] && NO_KILL_ECHO="echo " + TRAP+="${NO_KILL_ECHO} kill $! ||:;" + trap "$TRAP wait;" EXIT SIGINT SIGQUIT SIGTERM +} + +# Run program as daemon. Kill at exit +function daemon_killer { + bash -c "eval $* " & + last_killer +} + +function on_exit { + TRAP+="$* ||:;" + trap "$TRAP" EXIT SIGINT SIGQUIT SIGTERM +} + +function tmpl { + cp -v $1.tmpl $1 + perl -p -i -E 's/\${(.+)}/$ENV{$1}/g' $1 +} + +fi diff --git a/cloud/filestore/tools/testing/loadtest/test/load1.sh b/cloud/filestore/tools/testing/loadtest/test/load1.sh new file mode 100755 index 0000000000..8b62755673 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/load1.sh @@ -0,0 +1,3 @@ + +ffmpeg -f lavfi -i testsrc=duration=1:size=1080x1080:rate=30 -c:v rawvideo testsrc.mpg +ffmpeg -f rawvideo -pix_fmt rgb24 -video_size 1080x1080 -i testsrc.mpg -r 30 -c:v rawvideo -pix_fmt rgb24 -f caca - diff --git a/cloud/filestore/tools/testing/loadtest/test/load3.sh b/cloud/filestore/tools/testing/loadtest/test/load3.sh new file mode 100755 index 0000000000..1c1b963b09 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/load3.sh @@ -0,0 +1,6 @@ + +git clone --depth 1 --recursive https://github.com/google/leveldb +cd leveldb +cmake -G Ninja . +ninja +ctest diff --git a/cloud/filestore/tools/testing/loadtest/test/record.sh b/cloud/filestore/tools/testing/loadtest/test/record.sh new file mode 100755 index 0000000000..b09ec8f98f --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/record.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -ex + +export FS=record + +CUR_DIR=$(readlink -e $(dirname $0)) +. "${CUR_DIR}"/lib.sh + +export WORK_DIR=${CUR_DIR}/wrk +mkdir -p ${WORK_DIR} + +re_create_mount + +df -h +mount | grep ${HOME} ||: + +pushd ${MOUNT_POINT} + +# ${CUR_DIR}/load1.sh +# ${CUR_DIR}/load2.sh + ${CUR_DIR}/load3.sh + +find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/record_list.txt + +popd + + +sleep 15 + +ls -la ${LOGS_DIR}/filestore-server-profile-log.txt +cp ${LOGS_DIR}/filestore-server-profile-log.txt ${WORK_DIR} + + +umount ${MOUNT_POINT} ||: diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh b/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh new file mode 100755 index 0000000000..528e2d6122 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Write to local fs: +# REPLAY_ROOT=tmp/tmp NO_MOUNT=1 ./replay_fs.sh + +set -ex + +export FS=replay + +CUR_DIR=$(readlink -e $(dirname $0)) +. "${CUR_DIR}"/lib.sh + +export WORK_DIR=${CUR_DIR}/wrk +mkdir -p ${WORK_DIR} + +pushd ${ROOT_DIR} +ya make -r cloud/filestore/tools/testing/loadtest/bin +popd + +[ -z "${NO_MOUNT}" ] && re_create_mount + +df -h ||: +mount | grep ${HOME} ||: + +ls -la + +export REPLAY_ROOT=${REPLAY_ROOT=${MOUNT_POINT}/replayed} +tmpl ${CUR_DIR}/replay_fs.txt + +#LOADTEST_VERBOSE=${LT_VERBOSE=error} +LOADTEST_VERBOSE=${LT_VERBOSE=debug} +env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_fs.txt 2>&1 | tee ${WORK_DIR}/log_fs.log + +#popd + +pushd ${REPLAY_ROOT} +find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_fs_list.txt +popd + +[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: + +diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_fs_list.txt diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl b/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl new file mode 100644 index 0000000000..e2e93f3414 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl @@ -0,0 +1,18 @@ +Tests { + LoadTest { + Name: "smoke" + KeepFileStore: true + CreateFileStoreRequest: { + FileSystemId: "${FS}" + FolderId: "folder" + CloudId: "cloud" + BlocksCount: 10241024 + BlockSize: 4096 + } + ReplayFsSpec { + FileName: "${WORK_DIR}/filestore-server-profile-log.txt" + ReplayRoot: "${REPLAY_ROOT}" + } + IODepth: 2147483647 + } +} diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh new file mode 100755 index 0000000000..c25187ea72 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -ex + +export FS=replay + +CUR_DIR=$(readlink -e $(dirname $0)) +. "${CUR_DIR}"/lib.sh + +#ROOT_DIR=$(git rev-parse --show-toplevel) +export WORK_DIR=${CUR_DIR}/wrk +mkdir -p ${WORK_DIR} + +pushd ${ROOT_DIR} +ya make -r cloud/filestore/tools/testing/loadtest/bin +popd + +re_create_mount + +# no need to mount + +df -h ||: +mount | grep ${HOME} ||: + +tmpl ${CUR_DIR}/replay_grpc.txt + +#LOADTEST_VERBOSE=${VERBOSE=debug} +LOADTEST_VERBOSE=${VERBOSE=info} +env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_grpc.txt 2>&1 | tee ${WORK_DIR}/log_grpc.log + +pushd ${MOUNT_POINT} +find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_grpc_list.txt +popd + +[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: + +diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_grpc_list.txt diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl new file mode 100644 index 0000000000..dbb725a3dc --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl @@ -0,0 +1,17 @@ +Tests { + LoadTest { + Name: "smoke" + KeepFileStore: true + CreateFileStoreRequest: { + FileSystemId: "${FS}" + FolderId: "folder" + CloudId: "cloud" + BlocksCount: 10241024 + BlockSize: 4096 + } + ReplayGrpcSpec { + FileName: "${WORK_DIR}/filestore-server-profile-log.txt" + } + IODepth: 64 + } +} From 1af310a2b5d3bb959c7820053ba1d908b5a0aac9 Mon Sep 17 00:00:00 2001 From: proller Date: Mon, 30 Sep 2024 16:00:06 +0000 Subject: [PATCH 16/44] support links --- .../loadtest/lib/request_replay_fs.cpp | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 109248430c..d136e66c3a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -475,7 +475,7 @@ class TReplayRequestGeneratorFs final nodeName = KnownLogNodes[logRequest.GetNodeInfo().GetNodeId()].Name; } - auto parentpath = PathByNode(parentNode); + const auto parentpath = PathByNode(parentNode); if (nodeName.empty() && IsDir(Spec.GetReplayRoot() + parentpath)) { nodeName = @@ -523,7 +523,6 @@ class TReplayRequestGeneratorFs final TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; if (logRequest.GetNodeInfo().GetNodeId()) { NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = inode; - NodePath[inode] = relativePathName; } STORAGE_DEBUG( @@ -759,22 +758,33 @@ class TReplayRequestGeneratorFs final isDir = true; nodeid = MakeDirectoryRecursive(fullName); } break; - case NProto::E_LINK_NODE: - // TODO: NFs::HardLink(const TString &existingPath, const - // TString &newPath) NFs::SymLink(const TString &targetPath, - // const TString &linkPath) - break; + case NProto::E_LINK_NODE: { + // {"TimestampMcs":1727703903595285,"DurationMcs":2432,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":267,"NewNodeName":"pack-ebe666445578da0c6157f4172ad581cd731742ec.idx","Mode":292,"Type":3,"NodeId":274,"Size":245792}} + + const auto targetNode = + NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); + const auto targetFullName = + Spec.GetReplayRoot() + "/" + PathByNode(targetNode); + NFs::HardLink(targetFullName, fullName); + } break; + case NProto::E_SYMLINK_NODE: { + const auto targetNode = + NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); + const auto targetFullName = + Spec.GetReplayRoot() + "/" + PathByNode(targetNode); + NFs::SymLink(targetFullName, fullName); + } break; case NProto::E_SOCK_NODE: return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, Started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + MakeError(E_NOT_IMPLEMENTED, "sock not implemented")}); case NProto::E_INVALID_NODE: return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, Started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); + MakeError(E_NOT_IMPLEMENTED, "invalid not implemented")}); } if (!nodeid) { From c2e7c5c2b8a6822fc8f8cbeaf683d37893461ecc Mon Sep 17 00:00:00 2001 From: proller Date: Mon, 30 Sep 2024 19:26:13 +0000 Subject: [PATCH 17/44] Better --- .../tools/testing/loadtest/lib/request_replay_fs.cpp | 10 ++++++---- cloud/filestore/tools/testing/loadtest/test/load3.sh | 1 + cloud/filestore/tools/testing/loadtest/test/record.sh | 5 ++--- .../filestore/tools/testing/loadtest/test/replay_fs.sh | 2 -- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index d136e66c3a..686db9698e 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -519,17 +519,19 @@ class TReplayRequestGeneratorFs final OpenHandles[fh] = std::move(fileHandle); HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = fh; - const auto inode = - TFileStat{Spec.GetReplayRoot() + relativePathName}.INode; + const auto stat = + TFileStat{Spec.GetReplayRoot() + relativePathName}; + const auto inode = stat.INode; if (logRequest.GetNodeInfo().GetNodeId()) { NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = inode; NodePath[inode] = relativePathName; } STORAGE_DEBUG( - "open " << fh << "<-" << logRequest.GetNodeInfo().GetHandle() + "Open " << fh << " <- " << logRequest.GetNodeInfo().GetHandle() << " inode=" << inode << " known handles=" << HandlesLogToActual.size() - << " opened=" << OpenHandles.size()); + << " opened=" << OpenHandles.size() + << " size=" << stat.Size); } catch (const TFileError& error) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, diff --git a/cloud/filestore/tools/testing/loadtest/test/load3.sh b/cloud/filestore/tools/testing/loadtest/test/load3.sh index 1c1b963b09..9168d1f8c7 100755 --- a/cloud/filestore/tools/testing/loadtest/test/load3.sh +++ b/cloud/filestore/tools/testing/loadtest/test/load3.sh @@ -1,4 +1,5 @@ +# strace --follow-forks -- git clone --depth 1 --recursive https://github.com/google/leveldb 2>&1 | tee strace.log git clone --depth 1 --recursive https://github.com/google/leveldb cd leveldb cmake -G Ninja . diff --git a/cloud/filestore/tools/testing/loadtest/test/record.sh b/cloud/filestore/tools/testing/loadtest/test/record.sh index b09ec8f98f..e4b8d83b9b 100755 --- a/cloud/filestore/tools/testing/loadtest/test/record.sh +++ b/cloud/filestore/tools/testing/loadtest/test/record.sh @@ -12,7 +12,7 @@ mkdir -p ${WORK_DIR} re_create_mount -df -h +df -h ||: mount | grep ${HOME} ||: pushd ${MOUNT_POINT} @@ -31,5 +31,4 @@ sleep 15 ls -la ${LOGS_DIR}/filestore-server-profile-log.txt cp ${LOGS_DIR}/filestore-server-profile-log.txt ${WORK_DIR} - -umount ${MOUNT_POINT} ||: +[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh b/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh index 528e2d6122..bc74e7b1f3 100755 --- a/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh +++ b/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh @@ -31,8 +31,6 @@ tmpl ${CUR_DIR}/replay_fs.txt LOADTEST_VERBOSE=${LT_VERBOSE=debug} env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_fs.txt 2>&1 | tee ${WORK_DIR}/log_fs.log -#popd - pushd ${REPLAY_ROOT} find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_fs_list.txt popd From 42fa690398c8d3d8c70b82d0813b7f059e0c1f06 Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 1 Oct 2024 23:14:35 +0000 Subject: [PATCH 18/44] deduplicate code --- .../testing/loadtest/lib/request_replay.h | 236 ++++++++++ .../loadtest/lib/request_replay_fs.cpp | 404 +++--------------- .../loadtest/lib/request_replay_grpc.cpp | 288 +++---------- .../tools/testing/loadtest/test/record.sh | 6 +- 4 files changed, 348 insertions(+), 586 deletions(-) create mode 100644 cloud/filestore/tools/testing/loadtest/lib/request_replay.h diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h new file mode 100644 index 0000000000..f93116f0a0 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -0,0 +1,236 @@ +#pragma once + +#include "request.h" + +#include +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NLoadTest { + +using namespace NThreading; +using namespace NCloud::NFileStore::NClient; + +class IReplayRequestGenerator: public IRequestGenerator +{ +protected: + const NProto::TReplaySpec Spec; + TLog Log; + TString FileSystemId; + const NProto::THeaders Headers; + NClient::ISessionPtr Session; + + ui64 TimestampMcs{}; + TInstant Started; + +private: + THolder EventlogIterator; + TConstEventPtr EventPtr; + int EventMessageNumber = 0; + NProto::TProfileLogRecord* EventLogMessagePtr{}; + +public: + IReplayRequestGenerator( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) + { + Log = logging->CreateLog(Headers.GetClientId()); + + NEventLog::TOptions options; + options.FileName = Spec.GetFileName(); + options.SetForceStrongOrdering(true); // need this? + EventlogIterator = CreateIterator(options); + } + + bool InstantProcessQueue() override + { + return true; + } + + bool FailOnError() override + { + return false; + } + + void Advance() + { + EventPtr = EventlogIterator->Next(); + if (!EventPtr) { + return; + } + + EventLogMessagePtr = const_cast( + dynamic_cast( + EventPtr->GetProto())); + if (!EventLogMessagePtr) { + return; + } + + if (FileSystemId.empty()) { + FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; + } + + EventMessageNumber = EventLogMessagePtr->GetRequests().size(); + // DUMP(EventMessageNumber); + } + + bool HasNextRequest() override + { + if (!EventPtr) { + Advance(); + } + // DUMP(!!EventPtr); + return !!EventPtr; + } + + TInstant NextRequestAt() override + { + return TInstant::Max(); + } + + TFuture ExecuteNextRequest() override + { + // DUMP("ex"); + if (!HasNextRequest()) { + return MakeFuture(TCompletedRequest(true)); + } + + for (; EventPtr; Advance()) { + if (!EventLogMessagePtr) { + continue; + } + + for (; EventMessageNumber > 0;) { + auto request = + EventLogMessagePtr->GetRequests()[--EventMessageNumber]; + + { + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = TDuration::MicroSeconds( + timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); + + Sleep(slp); + } + Started = current; + } + + STORAGE_DEBUG( + "Processing message " + << EventMessageNumber + << " typename=" << request.GetTypeName() + << " type=" << request.GetRequestType() << " " + << " name=" << RequestName(request.GetRequestType()) + << " json=" << request.AsJSON()); + { + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest::ReadData: + return DoReadData(request); + case EFileStoreRequest::WriteData: + return DoWrite(request); + case EFileStoreRequest::CreateNode: + return DoCreateNode(request); + case EFileStoreRequest::RenameNode: + return DoRenameNode(request); + case EFileStoreRequest::UnlinkNode: + return DoUnlinkNode(request); + case EFileStoreRequest::CreateHandle: + return DoCreateHandle(request); + case EFileStoreRequest::DestroyHandle: + return DoDestroyHandle(request); + case EFileStoreRequest::GetNodeAttr: + return DoGetNodeAttr(request); + case EFileStoreRequest::AccessNode: + return DoAccessNode(request); + case EFileStoreRequest::ListNodes: + return DoListNodes(request); + + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(request); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + continue; + default: + STORAGE_INFO( + "Uninmplemented action=" + << action << " " + << RequestName(request.GetRequestType())); + continue; + } + } + } + } + STORAGE_INFO( + "Eventlog finished n=" << EventMessageNumber + << " ptr=" << !!EventPtr); + + return MakeFuture(TCompletedRequest(true)); + } + + virtual TFuture DoReadData( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoWrite( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoCreateNode( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoRenameNode( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoUnlinkNode( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoCreateHandle( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoDestroyHandle( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoGetNodeAttr( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoAccessNode( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoListNodes( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoAcquireLock( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; + virtual TFuture DoReleaseLock( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*unused*/) = 0; +}; + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 686db9698e..4755f1a69b 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -8,22 +8,20 @@ read/write with multiranges (now only first processed) #include "request.h" -#include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" -#include "cloud/filestore/tools/analytics/libs/event-log/dump.h" -#include "library/cpp/aio/aio.h" -#include "library/cpp/eventlog/iterator.h" -#include "library/cpp/testing/unittest/registar.h" -#include "util/folder/dirut.h" -#include "util/folder/path.h" -#include "util/system/fs.h" +#include "request_replay.h" #include +#include #include #include #include #include #include +#include +#include + +#include #include #include #include @@ -32,10 +30,12 @@ read/write with multiranges (now only first processed) #include #include #include +#include #include #include #include +#include namespace NCloud::NFileStore::NLoadTest { @@ -47,50 +47,30 @@ namespace { //////////////////////////////////////////////////////////////////////////////// class TReplayRequestGeneratorFs final - : public IRequestGenerator + : public IReplayRequestGenerator , public std::enable_shared_from_this { private: - const NProto::TReplaySpec Spec; - TString FileSystemId; - const NProto::THeaders Headers; - - TLog Log; - - ISessionPtr Session; - TVector> Actions; - TMutex StateLock; using THandleLog = ui64; using THandleLocal = ui64; + using TNodeLog = ui64; using TNodeLocal = ui64; std::atomic LastRequestId = 0; - THolder EventlogIterator; - TConstEventPtr EventPtr; - int EventMessageNumber = 0; - NProto::TProfileLogRecord* EventLogMessagePtr{}; - - static constexpr ui32 LockLength = 4096; - - const ui64 OwnerId = RandomNumber(100500u); - struct TNode { TString Name; - NProto::TNodeAttr Attrs; - TNodeLog ParentLog = 0; }; struct THandle { TString Path; - ui64 Handle = 0; }; THashMap Handles; @@ -105,8 +85,6 @@ class TReplayRequestGeneratorFs final THashMap OpenHandles; NAsyncIO::TAsyncIOService AsyncIO; - ui64 TimestampMcs{}; - TInstant Started; public: TReplayRequestGeneratorFs( @@ -115,24 +93,18 @@ class TReplayRequestGeneratorFs final ISessionPtr session, TString filesystemId, NProto::THeaders headers) - : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) - , Headers(std::move(headers)) - , Session(std::move(session)) + : IReplayRequestGenerator( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)) { - Log = logging->CreateLog(Headers.GetClientId()); - if (Spec.GetReplayRoot().empty()) { ythrow yexception() << "ReplayRoot is not defined"; } AsyncIO.Start(); - - NEventLog::TOptions options; - options.FileName = Spec.GetFileName(); - options.SetForceStrongOrdering(true); // need this? - EventlogIterator = CreateIterator(options); - TFsPath(Spec.GetReplayRoot()).MkDirs(); } ~TReplayRequestGeneratorFs() @@ -140,40 +112,6 @@ class TReplayRequestGeneratorFs final AsyncIO.Stop(); } - template - std::shared_ptr CreateRequest() - { - auto request = std::make_shared(); - request->SetFileSystemId(FileSystemId); - request->MutableHeaders()->CopyFrom(Headers); - - return request; - } - - template - void CheckResponse(const T& response) - { - if (HasError(response)) { - throw TServiceError(response.GetError()); - } - } - - TIntrusivePtr CreateCallContext() - { - return MakeIntrusive( - LastRequestId.fetch_add(1, std::memory_order_relaxed)); - } - - bool InstantProcessQueue() override - { - return true; - } - - bool FailOnError() override - { - return false; - } - TNodeLocal NodeIdMapped(const TNodeLog id) { if (const auto it = NodesLogToLocal.find(id); @@ -200,139 +138,10 @@ class TReplayRequestGeneratorFs final return 0; } - void Advance() - { - EventPtr = EventlogIterator->Next(); - if (!EventPtr) { - return; - } - - EventLogMessagePtr = const_cast( - dynamic_cast( - EventPtr->GetProto())); - if (!EventLogMessagePtr) { - return; - } - - if (FileSystemId.empty()) { - FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; - } - - EventMessageNumber = EventLogMessagePtr->GetRequests().size(); - } - - bool HasNextRequest() override - { - if (!EventPtr) { - Advance(); - } - return !!EventPtr; - } - - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - - NThreading::TFuture ExecuteNextRequest() override - { - if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest(true)); - } - - for (; EventPtr; Advance()) { - if (!EventLogMessagePtr) { - continue; - } - - for (; EventMessageNumber > 0;) { - auto request = - EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - - { - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - TimestampMcs = request.GetTimestampMcs(); - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - - if (timediff > diff.MicroSeconds()) { - auto slp = TDuration::MicroSeconds( - timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); - - Sleep(slp); - } - Started = current; - } - - STORAGE_DEBUG( - "Processing message " - << EventMessageNumber - << " typename=" << request.GetTypeName() - << " type=" << request.GetRequestType() << " " - << " name=" << RequestName(request.GetRequestType()) - << " json=" << request.AsJSON()); - { - const auto& action = request.GetRequestType(); - switch (static_cast(action)) { - case EFileStoreRequest::ReadData: - return DoReadData(request); - case EFileStoreRequest::WriteData: - return DoWrite(request); - case EFileStoreRequest::CreateNode: - return DoCreateNode(request); - case EFileStoreRequest::RenameNode: - return DoRenameNode(request); - case EFileStoreRequest::UnlinkNode: - return DoUnlinkNode(request); - case EFileStoreRequest::CreateHandle: - return DoCreateHandle(request); - case EFileStoreRequest::DestroyHandle: - return DoDestroyHandle(request); - case EFileStoreRequest::GetNodeAttr: - return DoGetNodeAttr(request); - case EFileStoreRequest::AccessNode: - return DoAccessNode(request); - case EFileStoreRequest::ListNodes: - return DoListNodes(request); - - // TODO: - case EFileStoreRequest::AcquireLock: - return DoAcquireLock(); - case EFileStoreRequest::ReleaseLock: - return DoReleaseLock(); - - case EFileStoreRequest::ReadBlob: - case EFileStoreRequest::WriteBlob: - case EFileStoreRequest::GenerateBlobIds: - case EFileStoreRequest::PingSession: - case EFileStoreRequest::Ping: - continue; - default: - STORAGE_INFO( - "Uninmplemented action=" - << action << " " - << RequestName(request.GetRequestType())); - continue; - } - } - } - } - STORAGE_INFO( - "Log finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); - - return MakeFuture(TCompletedRequest(true)); - } - private: TFuture DoAccessNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} @@ -424,6 +233,7 @@ class TReplayRequestGeneratorFs final TFuture DoCreateHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} @@ -517,7 +327,7 @@ class TReplayRequestGeneratorFs final MakeError(E_FAIL, "no filehandle")}); } - OpenHandles[fh] = std::move(fileHandle); + OpenHandles[fh] = fileHandle; HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = fh; const auto stat = TFileStat{Spec.GetReplayRoot() + relativePathName}; @@ -558,7 +368,8 @@ class TReplayRequestGeneratorFs final } TFuture DoReadData( - NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ @@ -595,14 +406,14 @@ class TReplayRequestGeneratorFs final } */ - TFileHandle FileHandle{fh.GetHandle()}; + TFileHandle fileHandle{fh.GetHandle()}; const auto future = AsyncIO.Read( - FileHandle, + fileHandle, {}, logRequest.GetRanges().cbegin()->GetBytes(), logRequest.GetRanges().cbegin()->GetOffset()); - FileHandle.Release(); + fileHandle.Release(); return future.Apply( [started = Started]([[maybe_unused]] const auto& future) mutable @@ -630,6 +441,7 @@ class TReplayRequestGeneratorFs final TFuture DoWrite( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} @@ -675,7 +487,7 @@ class TReplayRequestGeneratorFs final STORAGE_DEBUG( "Write to " << handle << " fh.length=" << fh.GetLength() << " fh.pos=" << fh.GetPosition()); - // TODO TEST USE AFTER FREE on buffer + // TODO(proller): TEST USE AFTER FREE on buffer TFileHandle FileHandle{fh.GetHandle()}; const auto writeFuture = AsyncIO.Write( // fh, @@ -714,6 +526,7 @@ class TReplayRequestGeneratorFs final TFuture DoCreateNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, @@ -741,8 +554,8 @@ class TReplayRequestGeneratorFs final bool isDir = false; switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { - // TODO: transform r.GetNodeInfo().GetMode() to correct open - // mode + // TODO(proller): transform r.GetNodeInfo().GetMode() to correct + // open mode TFileHandle fh( fullName, OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); @@ -806,6 +619,7 @@ class TReplayRequestGeneratorFs final } TFuture DoRenameNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, @@ -847,6 +661,7 @@ class TReplayRequestGeneratorFs final TFuture DoUnlinkNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} @@ -876,7 +691,7 @@ class TReplayRequestGeneratorFs final logRequest.GetNodeInfo().GetNodeName(); const auto unlinkres = NFs::Remove(fullName); STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); - // TODO : + // TODO(proller): // NodesLogToActual.erase(...) // NodePath.erase(...) return MakeFuture( @@ -885,6 +700,7 @@ class TReplayRequestGeneratorFs final TFuture DoDestroyHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} @@ -924,6 +740,7 @@ class TReplayRequestGeneratorFs final TFuture DoGetNodeAttr( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { if (Spec.GetNoRead()) { return MakeFuture(TCompletedRequest{ @@ -934,7 +751,7 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); - // TODO: by ParentNodeId + NodeName // + // TODO(proller): by ParentNodeId + NodeName // {"TimestampMcs":1726503153650998,"DurationMcs":7163,"RequestType":35,"ErrorCode":2147942422,"NodeInfo":{"NodeName":"security.capability","NewNodeName":"","NodeId":5,"Size":0}} // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} @@ -954,7 +771,7 @@ class TReplayRequestGeneratorFs final logRequest.GetNodeInfo().GetParentNodeId(); } - // TODO: can create and truncate to size here missing files + // TODO(proller): can create and truncate to size here missing files const auto nodeid = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); @@ -975,148 +792,29 @@ class TReplayRequestGeneratorFs final TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, Started, {})); } - TFuture DoAcquireLock() - { - TGuard guard(StateLock); - if (Handles.empty()) { - // return DoCreateHandle(); - } - - auto it = Handles.begin(); - while (it != Handles.end() && - (Locks.contains(it->first) || StagedLocks.contains(it->first))) - { - ++it; - } - - if (it == Handles.end()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - Started, - MakeError(E_CANCELLED, "cancelled")}); - } - - auto handle = it->first; - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto self = weak_from_this(); - return Session->AcquireLock(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleAcquireLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleAcquireLock( - ui64 handle, - const TFuture& future, - TInstant started) + TFuture DoAcquireLock( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*logRequest*/) override { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it == StagedLocks.end()) { - // nothing todo, file was removed - Y_ABORT_UNLESS(!Locks.contains(handle)); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } - - StagedLocks.erase(it); - - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - Locks.insert(handle); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "acquire lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_ACQUIRE_LOCK, started, error}; - } + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + Started, + MakeError(E_NOT_IMPLEMENTED, "invalid not implemented")}); } - TFuture DoReleaseLock() + TFuture DoReleaseLock( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*logRequest*/) override { - TGuard guard(StateLock); - if (Locks.empty()) { - return DoAcquireLock(); - } - - auto it = Locks.begin(); - auto handle = *it; - - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - Locks.erase(it); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto self = weak_from_this(); - return Session->ReleaseLock(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleReleaseLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_RELEASE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleReleaseLock( - ui64 handle, - const TFuture& future, - TInstant started) - { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it != StagedLocks.end()) { - StagedLocks.erase(it); - } - - try { - CheckResponse(future.GetValue()); - return {NProto::ACTION_RELEASE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "release lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_RELEASE_LOCK, started, error}; - } + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + Started, + MakeError(E_NOT_IMPLEMENTED, "invalid not implemented")}); } TFuture DoListNodes( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // json={"TimestampMcs":1726615510721016,"DurationMcs":3329,"RequestType":30,"ErrorCode":0,"NodeInfo":{"NodeId":164,"Size":10}} diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index 3a537c4fb5..ae63810527 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -9,18 +9,17 @@ compare log and actual result ( S_OK E_FS_NOENT ...) #include "request.h" -#include "cloud/filestore/libs/diagnostics/events/profile_events.ev.pb.h" -#include "cloud/filestore/tools/analytics/libs/event-log/dump.h" -#include "library/cpp/eventlog/iterator.h" -#include "library/cpp/testing/unittest/registar.h" -#include "util/system/fs.h" +#include "request_replay.h" #include +#include #include #include #include #include +#include + #include #include #include @@ -33,6 +32,7 @@ compare log and actual result ( S_OK E_FS_NOENT ...) #include #include +#include namespace NCloud::NFileStore::NLoadTest { @@ -44,20 +44,10 @@ namespace { //////////////////////////////////////////////////////////////////////////////// class TReplayRequestGeneratorGRPC final - : public IRequestGenerator + : public IReplayRequestGenerator , public std::enable_shared_from_this { private: - const NProto::TReplaySpec Spec; - TString FileSystemId; - const NProto::THeaders Headers; - - TLog Log; - - ISessionPtr Session; - - TVector> Actions; - TMutex StateLock; using THandleLog = ui64; @@ -69,11 +59,6 @@ class TReplayRequestGeneratorGRPC final std::atomic LastRequestId = 0; - THolder EventlogIterator; - TConstEventPtr EventPtr; - int EventMessageNumber = 0; - NProto::TProfileLogRecord* EventLogMessagePtr{}; - static constexpr ui32 LockLength = 4096; const ui64 OwnerId = RandomNumber(100500u); @@ -81,15 +66,11 @@ class TReplayRequestGeneratorGRPC final struct TNode { TString Name; - NProto::TNodeAttr Attrs; - - TNodeLog ParentLog = 0; }; struct THandle { TString Path; - ui64 Handle = 0; }; THashMap Handles; @@ -98,13 +79,8 @@ class TReplayRequestGeneratorGRPC final THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; THashMap NodePath{{RootNodeId, "/"}}; - THashMap KnownLogNodes; THashMap HandlesLogToActual; - // THashMap OpenHandles; - - ui64 TimestampMcs{}; - TInstant Started; public: TReplayRequestGeneratorGRPC( @@ -113,31 +89,14 @@ class TReplayRequestGeneratorGRPC final ISessionPtr session, TString filesystemId, NProto::THeaders headers) - : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) - , Headers(std::move(headers)) - , Session(std::move(session)) - { - Log = logging->CreateLog(Headers.GetClientId()); - - NEventLog::TOptions options; - options.FileName = Spec.GetFileName(); - options.SetForceStrongOrdering(true); // need this? - EventlogIterator = CreateIterator(options); - } - - ~TReplayRequestGeneratorGRPC() + : IReplayRequestGenerator( + std::move(spec), + std::move(logging), + std::move(session), + std::move(filesystemId), + std::move(headers)) {} - bool InstantProcessQueue() override - { - return true; - } - bool FailOnError() override - { - return false; - } - TNodeLocal NodeIdMapped(const TNodeLog id) { TGuard guard(StateLock); @@ -162,144 +121,18 @@ class TReplayRequestGeneratorGRPC final { return it->second; } + STORAGE_DEBUG( "handle not found " << id << " map size=" << HandlesLogToActual.size()); - return 0; - } - - void Advance() - { - EventPtr = EventlogIterator->Next(); - if (!EventPtr) { - return; - } - - EventLogMessagePtr = const_cast( - dynamic_cast( - EventPtr->GetProto())); - if (!EventLogMessagePtr) { - return; - } - - if (FileSystemId.empty()) { - FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; - } - - EventMessageNumber = EventLogMessagePtr->GetRequests().size(); - } - - bool HasNextRequest() override - { - if (!EventPtr) { - Advance(); - } - return !!EventPtr; - } - - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - - NThreading::TFuture ExecuteNextRequest() override - { - if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest(true)); - } - - for (; EventPtr; Advance()) { - if (!EventLogMessagePtr) { - continue; - } - for (; EventMessageNumber > 0;) { - auto request = - EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - - { - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - TimestampMcs = request.GetTimestampMcs(); - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - - if (timediff > diff.MicroSeconds()) { - auto slp = TDuration::MicroSeconds( - timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); - - Sleep(slp); - } - Started = current; - } - - STORAGE_DEBUG( - "Processing message " - << EventMessageNumber - << " typename=" << request.GetTypeName() - << " type=" << request.GetRequestType() << " " - << " name=" << RequestName(request.GetRequestType()) - << " json=" << request.AsJSON()); - - { - const auto& action = request.GetRequestType(); - switch (static_cast(action)) { - case EFileStoreRequest::ReadData: - return DoReadData(request); - case EFileStoreRequest::WriteData: - return DoWrite(request); - case EFileStoreRequest::CreateNode: - return DoCreateNode(request); - case EFileStoreRequest::RenameNode: - return DoRenameNode(request); - case EFileStoreRequest::UnlinkNode: - return DoUnlinkNode(request); - case EFileStoreRequest::CreateHandle: - return DoCreateHandle(request); - case EFileStoreRequest::DestroyHandle: - return DoDestroyHandle(request); - case EFileStoreRequest::GetNodeAttr: - return DoGetNodeAttr(request); - case EFileStoreRequest::AcquireLock: - return DoAcquireLock(); - case EFileStoreRequest::ReleaseLock: - return DoReleaseLock(); - - case EFileStoreRequest::AccessNode: - return DoAccessNode(request); - - case EFileStoreRequest::ReadBlob: - case EFileStoreRequest::WriteBlob: - case EFileStoreRequest::GenerateBlobIds: - case EFileStoreRequest::PingSession: - case EFileStoreRequest::Ping: - continue; - default: - STORAGE_INFO( - "Uninmplemented action=" - << action << " " - << RequestName(request.GetRequestType())); - continue; - } - } - } - } - STORAGE_INFO( - "Log finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); - - return MakeFuture(TCompletedRequest(true)); + return 0; } private: TFuture DoAccessNode( - [[maybe_unused]] const NCloud::NFileStore::NProto:: - TProfileLogRequestInfo& logRequest) + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*logRequest*/) override { // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} @@ -318,6 +151,7 @@ class TReplayRequestGeneratorGRPC final TFuture DoCreateHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} @@ -380,7 +214,7 @@ class TReplayRequestGeneratorGRPC final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); auto handle = response.GetHandle(); @@ -389,15 +223,15 @@ class TReplayRequestGeneratorGRPC final handle; } - NThreading::TFuture setAttr; + TFuture setAttr; if (InitialFileSize) { - static const int flags = + static const int Flags = ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); auto request = CreateRequest(); request->SetHandle(handle); request->SetNodeId(response.GetNodeAttr().GetId()); - request->SetFlags(flags); + request->SetFlags(Flags); request->MutableUpdate()->SetSize(InitialFileSize); setAttr = Session->SetNodeAttr( @@ -429,21 +263,9 @@ class TReplayRequestGeneratorGRPC final } } - static constexpr ui32 BlockSize = 4_KB; - static std::shared_ptr Acalloc(ui64 dataSize) - { - std::shared_ptr buffer = { - static_cast(aligned_alloc(BlockSize, dataSize)), - [](auto* p) - { - free(p); - }}; - memset(buffer.get(), 0, dataSize); - - return buffer; - } TFuture DoReadData( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { const auto started = TInstant::Now(); if (Spec.GetNoRead()) { @@ -493,7 +315,7 @@ class TReplayRequestGeneratorGRPC final TInstant started) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); return {NProto::ACTION_READ, started, response.GetError()}; } catch (const TServiceError& e) { @@ -507,7 +329,8 @@ class TReplayRequestGeneratorGRPC final } } - TString MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) + static TString + MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) { TStringBuilder ret; ret << "[\n" << start; @@ -519,6 +342,7 @@ class TReplayRequestGeneratorGRPC final TFuture DoWrite( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} @@ -581,7 +405,7 @@ class TReplayRequestGeneratorGRPC final TInstant started) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); return {NProto::ACTION_WRITE, started, response.GetError()}; @@ -597,23 +421,9 @@ class TReplayRequestGeneratorGRPC final } } - TString PathByNode(TNodeLocal nodeid) - { - if (const auto& it = NodePath.find(nodeid); it != NodePath.end()) { - return it->second; - } - return {}; - } - - static TNodeLocal MakeDirectoryRecursive(const TString& name) - { - NFs::MakeDirectoryRecursive(name); - const auto inode = TFileStat{name}.INode; - return inode; - } - TFuture DoCreateNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, @@ -650,13 +460,13 @@ class TReplayRequestGeneratorGRPC final logRequest.GetNodeInfo().GetMode()); break; case NProto::E_LINK_NODE: - // TODO: + // TODO(proller): // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); return MakeFuture(TCompletedRequest{}); break; case NProto::E_SYMLINK_NODE: - // TODO: + // TODO(proller): // request->MutableSymlink()->SetTargetPath(); break; case NProto::E_SOCK_NODE: @@ -705,7 +515,7 @@ class TReplayRequestGeneratorGRPC final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); // Nodes[name] = TNode{name, response.GetNode()}; if (response.GetNode().GetId()) { @@ -729,6 +539,7 @@ class TReplayRequestGeneratorGRPC final TFuture DoRenameNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, @@ -794,7 +605,7 @@ class TReplayRequestGeneratorGRPC final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; } catch (const TServiceError& e) { @@ -809,6 +620,7 @@ class TReplayRequestGeneratorGRPC final TFuture DoUnlinkNode( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} @@ -852,7 +664,7 @@ class TReplayRequestGeneratorGRPC final TInstant started) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; @@ -868,6 +680,7 @@ class TReplayRequestGeneratorGRPC final } TFuture DoDestroyHandle( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { // DestroyHandle 0.002475s S_OK {node_id=10, // handle=61465562388172112} @@ -906,6 +719,7 @@ class TReplayRequestGeneratorGRPC final TFuture DoGetNodeAttr( const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { const auto started = TInstant::Now(); if (Spec.GetNoRead()) { @@ -915,7 +729,7 @@ class TReplayRequestGeneratorGRPC final MakeError(E_PRECONDITION_FAILED, "disabled")}); } - // TODO: by parent + name // + // TODO(proller): by parent + name // // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, @@ -966,7 +780,7 @@ class TReplayRequestGeneratorGRPC final TInstant started) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); STORAGE_DEBUG("GetNodeAttr client completed"); CheckResponse(response); return {NProto::ACTION_GET_NODE_ATTR, started, {}}; @@ -987,7 +801,7 @@ class TReplayRequestGeneratorGRPC final TInstant started) { try { - auto response = future.GetValue(); + const auto& response = future.GetValue(); CheckResponse(response); return {NProto::ACTION_DESTROY_HANDLE, started, {}}; @@ -1002,9 +816,11 @@ class TReplayRequestGeneratorGRPC final } } - TFuture DoAcquireLock() + TFuture DoAcquireLock( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*logRequest*/) override { - // TODO: + // TODO(proller): return MakeFuture(TCompletedRequest{ NProto::ACTION_ACQUIRE_LOCK, Started, @@ -1087,7 +903,9 @@ class TReplayRequestGeneratorGRPC final } } - TFuture DoReleaseLock() + TFuture DoReleaseLock( + const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) + override { return MakeFuture(TCompletedRequest{ NProto::ACTION_RELEASE_LOCK, @@ -1096,7 +914,7 @@ class TReplayRequestGeneratorGRPC final TGuard guard(StateLock); if (Locks.empty()) { - return DoAcquireLock(); + return DoAcquireLock(logRequest); } auto it = Locks.begin(); @@ -1153,6 +971,16 @@ class TReplayRequestGeneratorGRPC final } } + TFuture DoListNodes( + const NCloud::NFileStore::NProto:: + TProfileLogRequestInfo& /*logRequest*/) override + { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_RELEASE_LOCK, + Started, + MakeError(E_NOT_IMPLEMENTED, "invalid not implemented")}); + } + template std::shared_ptr CreateRequest() { diff --git a/cloud/filestore/tools/testing/loadtest/test/record.sh b/cloud/filestore/tools/testing/loadtest/test/record.sh index e4b8d83b9b..e7754cca2b 100755 --- a/cloud/filestore/tools/testing/loadtest/test/record.sh +++ b/cloud/filestore/tools/testing/loadtest/test/record.sh @@ -17,9 +17,9 @@ mount | grep ${HOME} ||: pushd ${MOUNT_POINT} -# ${CUR_DIR}/load1.sh -# ${CUR_DIR}/load2.sh - ${CUR_DIR}/load3.sh +# ${CUR_DIR}/load1.sh ||: +# ${CUR_DIR}/load2.sh ||: + ${CUR_DIR}/load3.sh ||: find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/record_list.txt From 8f45bfd9226fe5f251334d39278b986c8cce1975 Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 1 Oct 2024 23:22:35 +0000 Subject: [PATCH 19/44] clean --- cloud/filestore/tools/testing/loadtest/lib/request_replay.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index f93116f0a0..6fb774fdb5 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -81,7 +81,6 @@ class IReplayRequestGenerator: public IRequestGenerator } EventMessageNumber = EventLogMessagePtr->GetRequests().size(); - // DUMP(EventMessageNumber); } bool HasNextRequest() override @@ -89,7 +88,6 @@ class IReplayRequestGenerator: public IRequestGenerator if (!EventPtr) { Advance(); } - // DUMP(!!EventPtr); return !!EventPtr; } @@ -100,7 +98,6 @@ class IReplayRequestGenerator: public IRequestGenerator TFuture ExecuteNextRequest() override { - // DUMP("ex"); if (!HasNextRequest()) { return MakeFuture(TCompletedRequest(true)); } From 2d3a2d177462a5a41502b1e3c69a5303250a5ef7 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 2 Oct 2024 23:17:50 +0000 Subject: [PATCH 20/44] Fix dirs --- .../testing/loadtest/lib/request_replay_fs.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 4755f1a69b..c79e647ac5 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -200,7 +200,8 @@ class TReplayRequestGeneratorFs final parentPath = PathByNode(parent); } if (parentPath.empty()) { - parentPath = "/__lost__/"; + static int LostNum = 0; + parentPath = "/__lost__/" + ToString(LostNum++) + "/"; } const auto name = parentPath + (it->second.Name.empty() ? ToString(nodeIdLog) @@ -831,11 +832,20 @@ class TReplayRequestGeneratorFs final << nodeid)}); } - // if (!Spec.GetNoWrite()) { - // CreateDirIfMissingByNodeLog(r.GetNodeInfo().GetNodeId()); - // } + if (!Spec.GetNoWrite()) { + CreateDirIfMissingByNodeLog(logRequest.GetNodeInfo().GetNodeId()); + } auto path = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); + if (NFs::Exists(path)) { + return MakeFuture(TCompletedRequest{ + NProto::ACTION_LIST_NODES, + Started, + MakeError( + E_NOT_FOUND, + TStringBuilder{} << "Local dir not found " << path)}); + } + // try { TFileHandle dir{path, RdOnly}; if (!dir.IsOpen()) { return MakeFuture(TCompletedRequest{ From ec9b25bd49c290c9c18493a5a75430c5a1fe5d9e Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 01:03:59 +0000 Subject: [PATCH 21/44] better names --- .../testing/loadtest/lib/request_replay_fs.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index c79e647ac5..af2b0dca54 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -200,13 +200,14 @@ class TReplayRequestGeneratorFs final parentPath = PathByNode(parent); } if (parentPath.empty()) { - static int LostNum = 0; - parentPath = "/__lost__/" + ToString(LostNum++) + "/"; + parentPath = + "/__lost__/" + ToString(it->second.ParentLog) + "/"; } - const auto name = parentPath + - (it->second.Name.empty() ? ToString(nodeIdLog) - : it->second.Name) + - "/"; + const auto name = + parentPath + + (it->second.Name.empty() ? "_nodeid_" + ToString(nodeIdLog) + : it->second.Name) + + "/"; const auto nodeId = MakeDirectoryRecursive(Spec.GetReplayRoot() + name); NodePath[nodeId] = name; From 944b95241ac5995ed195d537670cf0d21d159426 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 01:10:57 +0000 Subject: [PATCH 22/44] temporary remove grpc --- .../loadtest/lib/request_replay_grpc.cpp | 1026 +---------------- 1 file changed, 7 insertions(+), 1019 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp index ae63810527..1402055645 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp @@ -1,1028 +1,16 @@ -/* - -TODO: -create file/dir modes -create handle modes (now rw) -compare log and actual result ( S_OK E_FS_NOENT ...) - -*/ - #include "request.h" -#include "request_replay.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include +// TODO(proller): will be in next PR namespace NCloud::NFileStore::NLoadTest { - -using namespace NThreading; -using namespace NCloud::NFileStore::NClient; - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -class TReplayRequestGeneratorGRPC final - : public IReplayRequestGenerator - , public std::enable_shared_from_this -{ -private: - TMutex StateLock; - - using THandleLog = ui64; - using THandleLocal = ui64; - using TNodeLog = ui64; - using TNodeLocal = ui64; - - ui64 InitialFileSize = 0; - - std::atomic LastRequestId = 0; - - static constexpr ui32 LockLength = 4096; - - const ui64 OwnerId = RandomNumber(100500u); - - struct TNode - { - TString Name; - }; - - struct THandle - { - TString Path; - }; - - THashMap Handles; - THashSet Locks; - THashSet StagedLocks; - - THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; - THashMap NodePath{{RootNodeId, "/"}}; - - THashMap HandlesLogToActual; - -public: - TReplayRequestGeneratorGRPC( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) - : IReplayRequestGenerator( - std::move(spec), - std::move(logging), - std::move(session), - std::move(filesystemId), - std::move(headers)) - {} - - TNodeLocal NodeIdMapped(const TNodeLog id) - { - TGuard guard(StateLock); - - if (const auto it = NodesLogToLocal.find(id); - it != NodesLogToLocal.end()) - { - return it->second; - } - - STORAGE_DEBUG( - "node not found " << id << " map size=" << NodesLogToLocal.size()); - return 0; - } - - THandleLocal HandleIdMapped(const THandleLog id) - { - TGuard guard(StateLock); - - if (const auto it = HandlesLogToActual.find(id); - it != HandlesLogToActual.end()) - { - return it->second; - } - - STORAGE_DEBUG( - "handle not found " << id - << " map size=" << HandlesLogToActual.size()); - - return 0; - } - -private: - TFuture DoAccessNode( - const NCloud::NFileStore::NProto:: - TProfileLogRequestInfo& /*logRequest*/) override - { - // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} - - if (Spec.GetNoRead()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACCESS_NODE, - Started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACCESS_NODE, - Started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); - } - - TFuture DoCreateHandle( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - // json={"TimestampMcs":1726503808715698,"DurationMcs":2622,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":13882,"NodeName":"compile_commands.json.tmpdf020","Flags":38,"Mode":436,"NodeId":15553,"Handle":46923415058768564,"Size":0}} - // {"TimestampMcs":1725895168384258,"DurationMcs":2561,"RequestType":38,"ErrorCode":0,"NodeInfo":{"ParentNodeId":12527,"NodeName":"index.lock","Flags":15,"Mode":436,"NodeId":12584,"Handle":65382484937735195,"Size":0}} - // nfs CreateHandle 0.004161s S_OK {parent_node_id=65, - // node_name=ini, flags=14, mode=436, node_id=66, - // handle=11024287581389312, size=0} - - auto request = CreateRequest(); - auto name = logRequest.GetNodeInfo().GetNodeName(); - - const auto node = - NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{}); - } - request->SetNodeId(node); - request->SetName(logRequest.GetNodeInfo().GetNodeName()); - request->SetFlags(logRequest.GetNodeInfo().GetFlags()); - request->SetMode(logRequest.GetNodeInfo().GetMode()); - - STORAGE_DEBUG( - "open " << " handle=" << logRequest.GetNodeInfo().GetHandle() - << " flags=" << logRequest.GetNodeInfo().GetFlags() << " " - << HandleFlagsToString(logRequest.GetNodeInfo().GetFlags()) - << " mode=" << logRequest.GetNodeInfo().GetMode() - << " node=" << node << " <- " - << logRequest.GetNodeInfo().GetNodeId()); - - auto self = weak_from_this(); - const auto future = - Session->CreateHandle(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateHandle( - future, - name, - started, - logRequest); - } - - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - MakeError(E_FAIL, "cancelled")}); - }); - const auto& response = future.GetValueSync(); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - Started, - response.Error}); - } - - TFuture HandleCreateHandle( - const TFuture& future, - const TString& name, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - auto handle = response.GetHandle(); - with_lock (StateLock) { - HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = - handle; - } - - TFuture setAttr; - if (InitialFileSize) { - static const int Flags = - ProtoFlag(NProto::TSetNodeAttrRequest::F_SET_ATTR_SIZE); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetNodeId(response.GetNodeAttr().GetId()); - request->SetFlags(Flags); - request->MutableUpdate()->SetSize(InitialFileSize); - - setAttr = Session->SetNodeAttr( - CreateCallContext(), - std::move(request)); - } else { - setAttr = - NThreading::MakeFuture(NProto::TSetNodeAttrResponse()); - } - - return setAttr.Apply( - [=]([[maybe_unused]] const TFuture< - NProto::TSetNodeAttrResponse>& f) - { - return MakeFuture(TCompletedRequest{}); - // return HandleResizeAfterCreateHandle(f, name, started); - }); - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "create handle for %s has failed: %s", - name.Quote().c_str(), - FormatError(error).c_str()); - - return NThreading::MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_HANDLE, - started, - error}); - } - } - - TFuture DoReadData( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - const auto started = TInstant::Now(); - if (Spec.GetNoRead()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_READ, - started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - auto request = CreateRequest(); - const auto handle = - HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); - if (!handle) { - return MakeFuture(TCompletedRequest{}); - } - request->SetHandle(handle); - request->SetOffset(logRequest.GetRanges().cbegin()->GetOffset()); - request->SetLength(logRequest.GetRanges().cbegin()->GetBytes()); - - auto self = weak_from_this(); - return Session->ReadData(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - return TCompletedRequest{ - NProto::ACTION_READ, - started, - future.GetValue().GetError()}; - /* - if (auto ptr = self.lock()) { - return ptr->HandleRead( - future, - handleInfo, - started, - byteOffset); - } - */ - return TCompletedRequest{ - NProto::ACTION_READ, - started, - MakeError(E_FAIL, "cancelled")}; - }); - } - - TCompletedRequest HandleRead( - const TFuture& future, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - return {NProto::ACTION_READ, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "read for %s has failed: ", - // handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_READ, started, error}; - } - } - - static TString - MakeBuffer(ui64 bytes, ui64 offset = 0, const TString& start = {}) - { - TStringBuilder ret; - ret << "[\n" << start; - while (ret.size() < bytes) { - ret << " . " << ret.size() << " : " << offset + ret.size(); - } - return ret.substr(0, bytes); - } - - TFuture DoWrite( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} - - const auto started = TInstant::Now(); - if (Spec.GetNoWrite()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_WRITE, - started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - auto request = CreateRequest(); - const auto logHandle = logRequest.GetRanges(0).GetHandle(); - const auto handle = HandleIdMapped(logHandle); - - if (!handle) { - return MakeFuture(TCompletedRequest{}); - } - request->SetHandle(handle); - - const auto offset = logRequest.GetRanges(0).GetOffset(); - request->SetOffset(offset); - auto bytes = logRequest.GetRanges(0).GetBytes(); - - TString buffer; - - if (Spec.GetWriteRandom()) { - buffer = NUnitTest::RandomString(bytes, logHandle); - } else if (Spec.GetWriteEmpty()) { - buffer = TString{bytes, ' '}; - } else { - buffer = MakeBuffer( - bytes, - offset, - TStringBuilder{} << "handle=" << logHandle << " node=" - << logRequest.GetNodeInfo().GetNodeId() - << " bytes=" << bytes << " offset=" << offset); - } - - *request->MutableBuffer() = std::move(buffer); - - auto self = weak_from_this(); - return Session->WriteData(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleWrite(future, started); - } - - return TCompletedRequest{ - NProto::ACTION_WRITE, - started, - MakeError(E_FAIL, "cancelled")}; - }); - } - - TCompletedRequest HandleWrite( - const TFuture& future, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_WRITE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - /* - STORAGE_ERROR( - "write on %s has failed: %s", - handleInfo.Name.Quote().c_str(), - FormatError(error).c_str()); - */ - return {NProto::ACTION_WRITE, started, error}; - } - } - - TFuture DoCreateNode( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - // {"TimestampMcs":1725895166478218,"DurationMcs":6328,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":1,"NewNodeName":"home","Mode":509,"NodeId":12526,"Size":0}} - // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, - // new_node_name=home, mode=509, node_id=12526, size=0} - - if (Spec.GetNoWrite()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - Started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - auto request = CreateRequest(); - - const auto parentNode = - NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); - if (!parentNode) { - return MakeFuture(TCompletedRequest{}); - } - request->SetNodeId(parentNode); - auto name = logRequest.GetNodeInfo().GetNewNodeName(); - request->SetName(logRequest.GetNodeInfo().GetNewNodeName()); - - // request->SetGid(); - // request->SetUid(); - - switch (logRequest.GetNodeInfo().GetType()) { - case NProto::E_REGULAR_NODE: - request->MutableFile()->SetMode( - logRequest.GetNodeInfo().GetMode()); - break; - case NProto::E_DIRECTORY_NODE: - request->MutableDirectory()->SetMode( - logRequest.GetNodeInfo().GetMode()); - break; - case NProto::E_LINK_NODE: - // TODO(proller): - // request->MutableLink()->SetTargetNode(r.GetNodeInfo().Get...); - // request->MutableLink()->SetFollowerNodeName(r.GetNodeInfo().Get...); - return MakeFuture(TCompletedRequest{}); - break; - case NProto::E_SYMLINK_NODE: - // TODO(proller): - // request->MutableSymlink()->SetTargetPath(); - break; - case NProto::E_SOCK_NODE: - request->MutableSocket()->SetMode( - logRequest.GetNodeInfo().GetMode()); - break; - case NProto::E_INVALID_NODE: - return MakeFuture( - TCompletedRequest(NProto::ACTION_CREATE_NODE, Started, {})); - // Do not create files with invalid - // type - too hard to delete them - break; - } - Cerr << "createnoderec" << *request << "\n"; - auto self = weak_from_this(); - const auto future = - Session->CreateNode(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleCreateNode( - future, - name, - started, - logRequest); - } - - return TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - const auto& response = future.GetValueSync(); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_CREATE_NODE, - Started, - response.Error}); - } - - TCompletedRequest HandleCreateNode( - const TFuture& future, - const TString& name, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - // Nodes[name] = TNode{name, response.GetNode()}; - if (response.GetNode().GetId()) { - with_lock (StateLock) { - NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = - response.GetNode().GetId(); - } - } - - return {NProto::ACTION_CREATE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "create node %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_CREATE_NODE, started, error}; - } - } - - TFuture DoRenameNode( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - // {"TimestampMcs":895166000,"DurationMcs":2949,"RequestType":28,"NodeInfo":{"ParentNodeId":3,"NodeName":"HEAD.lock","NewParentNodeId":3,"NewNodeName":"HEAD"}} - // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, - // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} - // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); - if (Spec.GetNoRead()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - Started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - auto request = CreateRequest(); - const auto node = - NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - Started, - MakeError( - E_FAIL, - TStringBuilder{} - << "Log node " - << logRequest.GetNodeInfo().GetParentNodeId() - << "not found in mappiong")}); - } - request->SetNodeId(node); - request->SetName(logRequest.GetNodeInfo().GetNodeName()); - request->SetNewParentId( - NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId())); - request->SetNewName(logRequest.GetNodeInfo().GetNewNodeName()); - request->SetFlags(logRequest.GetNodeInfo().GetFlags()); - - auto self = weak_from_this(); - const auto future = - Session->RenameNode(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleRenameNode( - future, - started, - logRequest); - } - - return TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - started, - MakeError(E_INVALID_STATE, "invalid ptr")}; - }); - const auto& response = future.GetValueSync(); - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RENAME_NODE, - Started, - response.Error}); - } - - TCompletedRequest HandleRenameNode( - const TFuture& future, - TInstant started, - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - return {NProto::ACTION_RENAME_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "rename node %s has failed: %s", - logRequest.GetNodeInfo().GetNodeName().c_str(), - FormatError(error).c_str()); - return {NProto::ACTION_RENAME_NODE, started, error}; - } - } - - TFuture DoUnlinkNode( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - // UnlinkNode 0.002605s S_OK {parent_node_id=3, - // node_name=tfrgYZ1} - - if (Spec.GetNoWrite()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_REMOVE_NODE, - Started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - auto name = logRequest.GetNodeInfo().GetNodeName(); - auto request = CreateRequest(); - request->SetName(name); - const auto node = - NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{}); - } - request->SetNodeId(node); - auto self = weak_from_this(); - return Session->UnlinkNode(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleUnlinkNode(future, name, started); - } - - return TCompletedRequest{ - NProto::ACTION_REMOVE_NODE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleUnlinkNode( - const TFuture& future, - const TString& name, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_REMOVE_NODE, started, response.GetError()}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "unlink for %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_REMOVE_NODE, started, error}; - } - } - TFuture DoDestroyHandle( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - // DestroyHandle 0.002475s S_OK {node_id=10, - // handle=61465562388172112} - - auto name = logRequest.GetNodeInfo().GetNodeName(); - - auto request = CreateRequest(); - - const auto handle = - HandleIdMapped(logRequest.GetNodeInfo().GetHandle()); - if (!handle) { - return MakeFuture(TCompletedRequest{}); - } - with_lock (StateLock) { - HandlesLogToActual.erase(handle); - } - - request->SetHandle(handle); - - auto self = weak_from_this(); - return Session->DestroyHandle(CreateCallContext(), std::move(request)) - .Apply( - [=, started = Started, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleDestroyHandle(name, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_DESTROY_HANDLE, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TFuture DoGetNodeAttr( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - const auto started = TInstant::Now(); - if (Spec.GetNoRead()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError(E_PRECONDITION_FAILED, "disabled")}); - } - - // TODO(proller): by parent + name // - // {"TimestampMcs":1726615533406265,"DurationMcs":192,"RequestType":33,"ErrorCode":2147942402,"NodeInfo":{"ParentNodeId":17033,"NodeName":"CPackSourceConfig.cmake","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} - // {"TimestampMcs":240399000,"DurationMcs":163,"RequestType":33,"NodeInfo":{"ParentNodeId":3,"NodeName":"branches","Flags":0,"Mode":0,"NodeId":0,"Handle":0,"Size":0}} - // nfs GetNodeAttr 0.006847s S_OK {parent_node_id=1, - // node_name=freeminer, flags=0, mode=509, node_id=2, handle=0, size=0} - - auto request = CreateRequest(); - const auto node = - NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!node) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError( - E_NOT_FOUND, - TStringBuilder{} - << "Node missing in mapping " - << logRequest.GetNodeInfo().GetParentNodeId())}); - } - request->SetNodeId(node); - auto name = logRequest.GetNodeInfo().GetNodeName(); - request->SetName(logRequest.GetNodeInfo().GetNodeName()); - request->SetFlags(logRequest.GetNodeInfo().GetFlags()); - auto self = weak_from_this(); - STORAGE_DEBUG( - "GetNodeAttr client started name=" - << name << " node=" << node << " <- " - << logRequest.GetNodeInfo().GetParentNodeId() // why only parent? - << " request=" << *request); - return Session->GetNodeAttr(CreateCallContext(), std::move(request)) - .Apply( - [=, name = std::move(name)]( - const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleGetNodeAttr(name, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_GET_NODE_ATTR, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleGetNodeAttr( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - const auto& response = future.GetValue(); - STORAGE_DEBUG("GetNodeAttr client completed"); - CheckResponse(response); - return {NProto::ACTION_GET_NODE_ATTR, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "get node attr %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_GET_NODE_ATTR, started, error}; - } - } - - TCompletedRequest HandleDestroyHandle( - const TString& name, - const TFuture& future, - TInstant started) - { - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - return {NProto::ACTION_DESTROY_HANDLE, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "destroy handle %s has failed: %s", - name.c_str(), - FormatError(error).c_str()); - - return {NProto::ACTION_DESTROY_HANDLE, started, error}; - } - } - - TFuture DoAcquireLock( - const NCloud::NFileStore::NProto:: - TProfileLogRequestInfo& /*logRequest*/) override - { - // TODO(proller): - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - Started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); - - TGuard guard(StateLock); - if (Handles.empty()) { - // return DoCreateHandle(); - } - - auto started = TInstant::Now(); - auto it = Handles.begin(); - while (it != Handles.end() && - (Locks.contains(it->first) || StagedLocks.contains(it->first))) - { - ++it; - } - - if (it == Handles.end()) { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}); - } - - auto handle = it->first; - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto self = weak_from_this(); - return Session->AcquireLock(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleAcquireLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_ACQUIRE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleAcquireLock( - ui64 handle, - const TFuture& future, - TInstant started) - { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it == StagedLocks.end()) { - // nothing todo, file was removed - Y_ABORT_UNLESS(!Locks.contains(handle)); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } - - StagedLocks.erase(it); - - try { - const auto& response = future.GetValue(); - CheckResponse(response); - - Locks.insert(handle); - return {NProto::ACTION_ACQUIRE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "acquire lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_ACQUIRE_LOCK, started, error}; - } - } - - TFuture DoReleaseLock( - const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) - override - { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RELEASE_LOCK, - Started, - MakeError(E_NOT_IMPLEMENTED, "not implemented")}); - - TGuard guard(StateLock); - if (Locks.empty()) { - return DoAcquireLock(logRequest); - } - - auto it = Locks.begin(); - auto handle = *it; - - Y_ABORT_UNLESS(StagedLocks.insert(handle).second); - Locks.erase(it); - - auto request = CreateRequest(); - request->SetHandle(handle); - request->SetOwner(OwnerId); - request->SetLength(LockLength); - - auto started = TInstant::Now(); - auto self = weak_from_this(); - return Session->ReleaseLock(CreateCallContext(), std::move(request)) - .Apply( - [=](const TFuture& future) - { - if (auto ptr = self.lock()) { - return ptr->HandleReleaseLock(handle, future, started); - } - - return TCompletedRequest{ - NProto::ACTION_RELEASE_LOCK, - started, - MakeError(E_CANCELLED, "cancelled")}; - }); - } - - TCompletedRequest HandleReleaseLock( - ui64 handle, - const TFuture& future, - TInstant started) - { - TGuard guard(StateLock); - - auto it = StagedLocks.find(handle); - if (it != StagedLocks.end()) { - StagedLocks.erase(it); - } - - try { - CheckResponse(future.GetValue()); - return {NProto::ACTION_RELEASE_LOCK, started, {}}; - } catch (const TServiceError& e) { - auto error = MakeError(e.GetCode(), TString{e.GetMessage()}); - STORAGE_ERROR( - "release lock on %lu has failed: %s", - handle, - FormatError(error).c_str()); - - return {NProto::ACTION_RELEASE_LOCK, started, error}; - } - } - - TFuture DoListNodes( - const NCloud::NFileStore::NProto:: - TProfileLogRequestInfo& /*logRequest*/) override - { - return MakeFuture(TCompletedRequest{ - NProto::ACTION_RELEASE_LOCK, - Started, - MakeError(E_NOT_IMPLEMENTED, "invalid not implemented")}); - } - - template - std::shared_ptr CreateRequest() - { - auto request = std::make_shared(); - request->SetFileSystemId(FileSystemId); - request->MutableHeaders()->CopyFrom(Headers); - - return request; - } - - template - void CheckResponse(const T& response) - { - if (HasError(response)) { - throw TServiceError(response.GetError()); - } - } - - TIntrusivePtr CreateCallContext() - { - return MakeIntrusive( - LastRequestId.fetch_add(1, std::memory_order_relaxed)); - } -}; - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, - NProto::THeaders headers) + NProto::TReplaySpec /*spec*/, + ILoggingServicePtr /*logging*/, + NClient::ISessionPtr /*session*/, + TString /*filesystemId*/, + NProto::THeaders /*headers*/) { - return std::make_shared( - std::move(spec), - std::move(logging), - std::move(session), - std::move(filesystemId), - std::move(headers)); + return {}; } } // namespace NCloud::NFileStore::NLoadTest From 940f7b18068641843c3876f0d4a94d81a060b516 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 18:47:19 +0000 Subject: [PATCH 23/44] clean --- .../tools/testing/loadtest/test/lib.sh | 83 ------------------- .../tools/testing/loadtest/test/load1.sh | 3 - .../tools/testing/loadtest/test/load3.sh | 7 -- .../tools/testing/loadtest/test/record.sh | 34 -------- .../tools/testing/loadtest/test/replay_fs.sh | 40 --------- .../testing/loadtest/test/replay_fs.txt.tmpl | 18 ---- .../testing/loadtest/test/replay_grpc.sh | 37 --------- .../loadtest/test/replay_grpc.txt.tmpl | 17 ---- 8 files changed, 239 deletions(-) delete mode 100644 cloud/filestore/tools/testing/loadtest/test/lib.sh delete mode 100755 cloud/filestore/tools/testing/loadtest/test/load1.sh delete mode 100755 cloud/filestore/tools/testing/loadtest/test/load3.sh delete mode 100755 cloud/filestore/tools/testing/loadtest/test/record.sh delete mode 100755 cloud/filestore/tools/testing/loadtest/test/replay_fs.sh delete mode 100644 cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl delete mode 100755 cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh delete mode 100644 cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl diff --git a/cloud/filestore/tools/testing/loadtest/test/lib.sh b/cloud/filestore/tools/testing/loadtest/test/lib.sh deleted file mode 100644 index a1f92e53a9..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/lib.sh +++ /dev/null @@ -1,83 +0,0 @@ - -if [ -z ${CONFIGURED} ]; then -export CONFIGURED=1 - -[ -z "${VERBOSE=${V}}" ] && set +x -[ -n "${VERBOSE=${V}}" ] && set -x -[ -z "${KEEP}" ] && set -e - -#export ARCADIA_ROOT=${ARCADIA_ROOT:=`ya dump root || echo ~/arcadia/`} - -ROOT_DIR=$(git rev-parse --show-toplevel) - - -# CUR_DIR=${CUR_DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} - -# YA_MAKE_ADD=--sanitize=address -# YA_MAKE_ADD=--sanitize=memory -# YA_MAKE_ADD=--sanitize=thread -# YA_MAKE_ADD=--sanitize=undefined -# YA_MAKE_ADD=--sanitize=leak -# YA_MAKE_ADD=-r - -#YA_MAKE_ADD=${YA_MAKE_ADD=-r} -#YA_MAKE=${YA_MAKE="ya make ${YA_MAKE_ADD}"} # " - -REBUILD=${REBUILD=1} - -# STARTER='ya tool gdb -ex run --args' -# STARTER='ya tool gdb -ex run -ex backtrace --args' -# STARTER='GRPC_VERBOSITY=info GRPC_TRACE=tcp,http,api' - - - - -function re_create_mount { -pushd ${ROOT_DIR} -ya make -r cloud/filestore/buildall -popd - -pushd ${ROOT_DIR}/cloud/filestore/bin - -. initctl.sh -umount ${MOUNT_POINT} ||: -. initctl.sh stop - -. initctl.sh format initialize -. initctl.sh create startendpoint -. initctl.sh start -. initctl.sh mount - -sleep 3 -echo mountpoint = ${MOUNT_POINT} -popd - -} - - -# === Library section - -# Add pid of last runned program to "kill on exit" list -function last_killer { - [ -n "${NO_KILL}" ] && NO_KILL_ECHO="echo " - TRAP+="${NO_KILL_ECHO} kill $! ||:;" - trap "$TRAP wait;" EXIT SIGINT SIGQUIT SIGTERM -} - -# Run program as daemon. Kill at exit -function daemon_killer { - bash -c "eval $* " & - last_killer -} - -function on_exit { - TRAP+="$* ||:;" - trap "$TRAP" EXIT SIGINT SIGQUIT SIGTERM -} - -function tmpl { - cp -v $1.tmpl $1 - perl -p -i -E 's/\${(.+)}/$ENV{$1}/g' $1 -} - -fi diff --git a/cloud/filestore/tools/testing/loadtest/test/load1.sh b/cloud/filestore/tools/testing/loadtest/test/load1.sh deleted file mode 100755 index 8b62755673..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/load1.sh +++ /dev/null @@ -1,3 +0,0 @@ - -ffmpeg -f lavfi -i testsrc=duration=1:size=1080x1080:rate=30 -c:v rawvideo testsrc.mpg -ffmpeg -f rawvideo -pix_fmt rgb24 -video_size 1080x1080 -i testsrc.mpg -r 30 -c:v rawvideo -pix_fmt rgb24 -f caca - diff --git a/cloud/filestore/tools/testing/loadtest/test/load3.sh b/cloud/filestore/tools/testing/loadtest/test/load3.sh deleted file mode 100755 index 9168d1f8c7..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/load3.sh +++ /dev/null @@ -1,7 +0,0 @@ - -# strace --follow-forks -- git clone --depth 1 --recursive https://github.com/google/leveldb 2>&1 | tee strace.log -git clone --depth 1 --recursive https://github.com/google/leveldb -cd leveldb -cmake -G Ninja . -ninja -ctest diff --git a/cloud/filestore/tools/testing/loadtest/test/record.sh b/cloud/filestore/tools/testing/loadtest/test/record.sh deleted file mode 100755 index e7754cca2b..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/record.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export FS=record - -CUR_DIR=$(readlink -e $(dirname $0)) -. "${CUR_DIR}"/lib.sh - -export WORK_DIR=${CUR_DIR}/wrk -mkdir -p ${WORK_DIR} - -re_create_mount - -df -h ||: -mount | grep ${HOME} ||: - -pushd ${MOUNT_POINT} - -# ${CUR_DIR}/load1.sh ||: -# ${CUR_DIR}/load2.sh ||: - ${CUR_DIR}/load3.sh ||: - -find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/record_list.txt - -popd - - -sleep 15 - -ls -la ${LOGS_DIR}/filestore-server-profile-log.txt -cp ${LOGS_DIR}/filestore-server-profile-log.txt ${WORK_DIR} - -[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh b/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh deleted file mode 100755 index bc74e7b1f3..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/replay_fs.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Write to local fs: -# REPLAY_ROOT=tmp/tmp NO_MOUNT=1 ./replay_fs.sh - -set -ex - -export FS=replay - -CUR_DIR=$(readlink -e $(dirname $0)) -. "${CUR_DIR}"/lib.sh - -export WORK_DIR=${CUR_DIR}/wrk -mkdir -p ${WORK_DIR} - -pushd ${ROOT_DIR} -ya make -r cloud/filestore/tools/testing/loadtest/bin -popd - -[ -z "${NO_MOUNT}" ] && re_create_mount - -df -h ||: -mount | grep ${HOME} ||: - -ls -la - -export REPLAY_ROOT=${REPLAY_ROOT=${MOUNT_POINT}/replayed} -tmpl ${CUR_DIR}/replay_fs.txt - -#LOADTEST_VERBOSE=${LT_VERBOSE=error} -LOADTEST_VERBOSE=${LT_VERBOSE=debug} -env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_fs.txt 2>&1 | tee ${WORK_DIR}/log_fs.log - -pushd ${REPLAY_ROOT} -find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_fs_list.txt -popd - -[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: - -diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_fs_list.txt diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl b/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl deleted file mode 100644 index e2e93f3414..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/replay_fs.txt.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -Tests { - LoadTest { - Name: "smoke" - KeepFileStore: true - CreateFileStoreRequest: { - FileSystemId: "${FS}" - FolderId: "folder" - CloudId: "cloud" - BlocksCount: 10241024 - BlockSize: 4096 - } - ReplayFsSpec { - FileName: "${WORK_DIR}/filestore-server-profile-log.txt" - ReplayRoot: "${REPLAY_ROOT}" - } - IODepth: 2147483647 - } -} diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh deleted file mode 100755 index c25187ea72..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -export FS=replay - -CUR_DIR=$(readlink -e $(dirname $0)) -. "${CUR_DIR}"/lib.sh - -#ROOT_DIR=$(git rev-parse --show-toplevel) -export WORK_DIR=${CUR_DIR}/wrk -mkdir -p ${WORK_DIR} - -pushd ${ROOT_DIR} -ya make -r cloud/filestore/tools/testing/loadtest/bin -popd - -re_create_mount - -# no need to mount - -df -h ||: -mount | grep ${HOME} ||: - -tmpl ${CUR_DIR}/replay_grpc.txt - -#LOADTEST_VERBOSE=${VERBOSE=debug} -LOADTEST_VERBOSE=${VERBOSE=info} -env LD_LIBRARY_PATH=${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin ${GDB} ${ROOT_DIR}/cloud/filestore/tools/testing/loadtest/bin/filestore-loadtest --verbose ${LOADTEST_VERBOSE} --tests-config ${CUR_DIR}/replay_grpc.txt 2>&1 | tee ${WORK_DIR}/log_grpc.log - -pushd ${MOUNT_POINT} -find . -type f -iname "*" -printf "%h/%f %s \n" | sort | tee ${WORK_DIR}/replay_grpc_list.txt -popd - -[ -z "${KEEP_MOUNT}" ] && umount ${MOUNT_POINT} ||: - -diff ${WORK_DIR}/record_list.txt ${WORK_DIR}/replay_grpc_list.txt diff --git a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl b/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl deleted file mode 100644 index dbb725a3dc..0000000000 --- a/cloud/filestore/tools/testing/loadtest/test/replay_grpc.txt.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -Tests { - LoadTest { - Name: "smoke" - KeepFileStore: true - CreateFileStoreRequest: { - FileSystemId: "${FS}" - FolderId: "folder" - CloudId: "cloud" - BlocksCount: 10241024 - BlockSize: 4096 - } - ReplayGrpcSpec { - FileName: "${WORK_DIR}/filestore-server-profile-log.txt" - } - IODepth: 64 - } -} From ed9f9d55a1f76953dca7a3f7131edc12c243e5a4 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 18:53:05 +0000 Subject: [PATCH 24/44] clean --- cloud/filestore/bin/initctl.sh | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/cloud/filestore/bin/initctl.sh b/cloud/filestore/bin/initctl.sh index f839f7401d..c1f50068ae 100755 --- a/cloud/filestore/bin/initctl.sh +++ b/cloud/filestore/bin/initctl.sh @@ -7,26 +7,18 @@ find_bin_dir() { } BIN_DIR=`find_bin_dir` -export LOGS_DIR=$BIN_DIR/data +LOGS_DIR=$BIN_DIR/data -export SERVER_PORT=${SERVER_PORT:-9021} -export VHOST_PORT=${VHOST_PORT:-9022} +SERVER_PORT=${SERVER_PORT:-9021} +VHOST_PORT=${VHOST_PORT:-9022} -export FS=${FS:-"nfs"} -export SHARD_COUNT=${SHARD_COUNT:-0} -export BLOCK_SIZE=${BLOCK_SIZE:-4096} -export MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"} -export VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock} +FS=${FS:-"nfs"} +SHARD_COUNT=${SHARD_COUNT:-0} +BLOCK_SIZE=${BLOCK_SIZE:-4096} +MOUNT_POINT=${MOUNT_POINT:-"$HOME/$FS"} +VHOST_SOCKET_PATH=${VHOST_SOCKET_PATH:-/tmp/vhost.sock} -export PID_FILE=$BIN_DIR/pids.txt - -if [ -z "$FILESTORE_APPS_PATH" ]; then - REPO_ROOT=$BIN_DIR/../../.. - FILESTORE_APPS_PATH=cloud/filestore/buildall/cloud/filestore/apps - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/client - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/server - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$REPO_ROOT/$FILESTORE_APPS_PATH/vhost -fi +PID_FILE=$BIN_DIR/pids.txt ################################################################################ # STOP/KILL From 64d954c96b474bedbe8d02c4f3ad7bbdcb82de66 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 20:38:09 +0000 Subject: [PATCH 25/44] Clean --- .../tools/testing/loadtest/lib/request.h | 7 ------- .../testing/loadtest/lib/request_replay_grpc.cpp | 16 ---------------- .../tools/testing/loadtest/lib/test.cpp | 9 --------- .../filestore/tools/testing/loadtest/lib/ya.make | 1 - 4 files changed, 33 deletions(-) delete mode 100644 cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 74abd8abdc..551d0818a9 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -83,11 +83,4 @@ IRequestGeneratorPtr CreateReplayRequestGeneratorFs( TString filesystemId, NProto::THeaders headers); -IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - NClient::ISessionPtr session, - TString filesystemId, - NProto::THeaders headers); - } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp deleted file mode 100644 index 1402055645..0000000000 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_grpc.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "request.h" - -// TODO(proller): will be in next PR - -namespace NCloud::NFileStore::NLoadTest { -IRequestGeneratorPtr CreateReplayRequestGeneratorGRPC( - NProto::TReplaySpec /*spec*/, - ILoggingServicePtr /*logging*/, - NClient::ISessionPtr /*session*/, - TString /*filesystemId*/, - NProto::THeaders /*headers*/) -{ - return {}; -} - -} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 7c85776a17..682e0a7215 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -540,15 +540,6 @@ class TLoadTest final FileSystemId, headers); break; - - case NProto::TLoadTest::kReplayGrpcSpec: - RequestGenerator = CreateReplayRequestGeneratorGRPC( - Config.GetReplayGrpcSpec(), - Logging, - Session, - FileSystemId, - headers); - break; default: ythrow yexception() << MakeTestTag() diff --git a/cloud/filestore/tools/testing/loadtest/lib/ya.make b/cloud/filestore/tools/testing/loadtest/lib/ya.make index 5ec90fbf77..b35058ef35 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/ya.make +++ b/cloud/filestore/tools/testing/loadtest/lib/ya.make @@ -7,7 +7,6 @@ SRCS( request_data.cpp request_index.cpp request_replay_fs.cpp - request_replay_grpc.cpp test.cpp ) From 9919e94e8b90ba93a1d9f24af5a7a450de03994b Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 3 Oct 2024 23:25:12 +0000 Subject: [PATCH 26/44] clean --- .../tools/testing/loadtest/lib/request.h | 4 +- .../testing/loadtest/lib/request_replay.cpp | 171 ++++++++++++++++ .../testing/loadtest/lib/request_replay.h | 182 ++---------------- .../loadtest/lib/request_replay_fs.cpp | 19 +- .../tools/testing/loadtest/lib/test.cpp | 5 +- .../tools/testing/loadtest/lib/ya.make | 1 + 6 files changed, 201 insertions(+), 181 deletions(-) create mode 100644 cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 551d0818a9..96ecda3b76 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -49,12 +49,12 @@ struct IRequestGenerator virtual TInstant NextRequestAt() = 0; virtual NThreading::TFuture ExecuteNextRequest() = 0; - virtual bool InstantProcessQueue() + virtual bool ShouldInstantProcessQueue() { return false; } - virtual bool FailOnError() + virtual bool ShouldFailOnError() { return true; } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp new file mode 100644 index 0000000000..5a57bfd7e4 --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -0,0 +1,171 @@ +#include "request_replay.h" + +#include +#include +#include +#include + +#include +#include +#include + +namespace NCloud::NFileStore::NLoadTest { +IReplayRequestGenerator::IReplayRequestGenerator( + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString filesystemId, + NProto::THeaders headers) + : Spec(std::move(spec)) + , FileSystemId(std::move(filesystemId)) + , Headers(std::move(headers)) + , Session(std::move(session)) +{ + Log = logging->CreateLog(Headers.GetClientId()); + + NEventLog::TOptions options; + options.FileName = Spec.GetFileName(); + options.SetForceStrongOrdering(true); // need this? + CurrentEvent = CreateIterator(options); +} + +bool IReplayRequestGenerator::HasNextRequest() +{ + if (!EventPtr) { + Advance(); + } + return !!EventPtr; +} + +TInstant IReplayRequestGenerator::NextRequestAt() +{ + return TInstant::Max(); +} + +bool IReplayRequestGenerator::ShouldInstantProcessQueue() +{ + return true; +} + +bool IReplayRequestGenerator::ShouldFailOnError() +{ + return false; +} + +void IReplayRequestGenerator::Advance() +{ + EventPtr = CurrentEvent->Next(); + if (!EventPtr) { + return; + } + + MessagePtr = + dynamic_cast(EventPtr->GetProto()); + + if (!MessagePtr) { + return; + } + + if (FileSystemId.empty()) { + FileSystemId = TString{MessagePtr->GetFileSystemId()}; + } + + EventMessageNumber = MessagePtr->GetRequests().size(); +} + +NThreading::TFuture +IReplayRequestGenerator::ExecuteNextRequest() +{ + if (!HasNextRequest()) { + return MakeFuture(TCompletedRequest(true)); + } + + for (; EventPtr; Advance()) { + if (!MessagePtr) { + continue; + } + + for (; EventMessageNumber > 0;) { + auto request = MessagePtr->GetRequests()[--EventMessageNumber]; + + { + auto timediff = (request.GetTimestampMcs() - TimestampMcs) * + Spec.GetTimeScale(); + TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { + timediff = 0; + } + const auto current = TInstant::Now(); + auto diff = current - Started; + + if (timediff > diff.MicroSeconds()) { + auto slp = + TDuration::MicroSeconds(timediff - diff.MicroSeconds()); + STORAGE_DEBUG( + "sleep=" << slp << " timediff=" << timediff + << " diff=" << diff); + + Sleep(slp); + } + Started = current; + } + + STORAGE_DEBUG( + "Processing message " + << EventMessageNumber << " typename=" << request.GetTypeName() + << " type=" << request.GetRequestType() << " " + << " name=" << RequestName(request.GetRequestType()) + << " json=" << request.AsJSON()); + { + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest::ReadData: + return DoReadData(request); + case EFileStoreRequest::WriteData: + return DoWrite(request); + case EFileStoreRequest::CreateNode: + return DoCreateNode(request); + case EFileStoreRequest::RenameNode: + return DoRenameNode(request); + case EFileStoreRequest::UnlinkNode: + return DoUnlinkNode(request); + case EFileStoreRequest::CreateHandle: + return DoCreateHandle(request); + case EFileStoreRequest::DestroyHandle: + return DoDestroyHandle(request); + case EFileStoreRequest::GetNodeAttr: + return DoGetNodeAttr(request); + case EFileStoreRequest::AccessNode: + return DoAccessNode(request); + case EFileStoreRequest::ListNodes: + return DoListNodes(request); + + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(request); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + continue; + default: + STORAGE_INFO( + "Uninmplemented action=" + << action << " " + << RequestName(request.GetRequestType())); + continue; + } + } + } + } + STORAGE_INFO( + "Profile log finished n=" << EventMessageNumber + << " ptr=" << !!EventPtr); + + return MakeFuture(TCompletedRequest(true)); +} + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index 6fb774fdb5..dd55e19cf9 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -3,10 +3,8 @@ #include "request.h" #include -#include -#include -#include +#include #include namespace NCloud::NFileStore::NLoadTest { @@ -14,23 +12,25 @@ namespace NCloud::NFileStore::NLoadTest { using namespace NThreading; using namespace NCloud::NFileStore::NClient; +//////////////////////////////////////////////////////////////////////////////// + class IReplayRequestGenerator: public IRequestGenerator { protected: - const NProto::TReplaySpec Spec; + const ::NCloud::NFileStore::NProto::TReplaySpec Spec; TLog Log; TString FileSystemId; - const NProto::THeaders Headers; + const ::NCloud::NFileStore::NProto::THeaders Headers; NClient::ISessionPtr Session; ui64 TimestampMcs{}; TInstant Started; private: - THolder EventlogIterator; + THolder CurrentEvent; TConstEventPtr EventPtr; - int EventMessageNumber = 0; - NProto::TProfileLogRecord* EventLogMessagePtr{}; + ui64 EventMessageNumber = 0; + const NProto::TProfileLogRecord* MessagePtr{}; public: IReplayRequestGenerator( @@ -38,159 +38,19 @@ class IReplayRequestGenerator: public IRequestGenerator ILoggingServicePtr logging, NClient::ISessionPtr session, TString filesystemId, - NProto::THeaders headers) - : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) - , Headers(std::move(headers)) - , Session(std::move(session)) - { - Log = logging->CreateLog(Headers.GetClientId()); - - NEventLog::TOptions options; - options.FileName = Spec.GetFileName(); - options.SetForceStrongOrdering(true); // need this? - EventlogIterator = CreateIterator(options); - } - - bool InstantProcessQueue() override - { - return true; - } - - bool FailOnError() override - { - return false; - } - - void Advance() - { - EventPtr = EventlogIterator->Next(); - if (!EventPtr) { - return; - } - - EventLogMessagePtr = const_cast( - dynamic_cast( - EventPtr->GetProto())); - if (!EventLogMessagePtr) { - return; - } - - if (FileSystemId.empty()) { - FileSystemId = TString{EventLogMessagePtr->GetFileSystemId()}; - } - - EventMessageNumber = EventLogMessagePtr->GetRequests().size(); - } - - bool HasNextRequest() override - { - if (!EventPtr) { - Advance(); - } - return !!EventPtr; - } - - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - - TFuture ExecuteNextRequest() override - { - if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest(true)); - } - - for (; EventPtr; Advance()) { - if (!EventLogMessagePtr) { - continue; - } - - for (; EventMessageNumber > 0;) { - auto request = - EventLogMessagePtr->GetRequests()[--EventMessageNumber]; - - { - auto timediff = (request.GetTimestampMcs() - TimestampMcs) * - Spec.GetTimeScale(); - TimestampMcs = request.GetTimestampMcs(); - if (timediff > 1000000) { - timediff = 0; - } - const auto current = TInstant::Now(); - auto diff = current - Started; - - if (timediff > diff.MicroSeconds()) { - auto slp = TDuration::MicroSeconds( - timediff - diff.MicroSeconds()); - STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); - - Sleep(slp); - } - Started = current; - } - - STORAGE_DEBUG( - "Processing message " - << EventMessageNumber - << " typename=" << request.GetTypeName() - << " type=" << request.GetRequestType() << " " - << " name=" << RequestName(request.GetRequestType()) - << " json=" << request.AsJSON()); - { - const auto& action = request.GetRequestType(); - switch (static_cast(action)) { - case EFileStoreRequest::ReadData: - return DoReadData(request); - case EFileStoreRequest::WriteData: - return DoWrite(request); - case EFileStoreRequest::CreateNode: - return DoCreateNode(request); - case EFileStoreRequest::RenameNode: - return DoRenameNode(request); - case EFileStoreRequest::UnlinkNode: - return DoUnlinkNode(request); - case EFileStoreRequest::CreateHandle: - return DoCreateHandle(request); - case EFileStoreRequest::DestroyHandle: - return DoDestroyHandle(request); - case EFileStoreRequest::GetNodeAttr: - return DoGetNodeAttr(request); - case EFileStoreRequest::AccessNode: - return DoAccessNode(request); - case EFileStoreRequest::ListNodes: - return DoListNodes(request); - - case EFileStoreRequest::AcquireLock: - return DoAcquireLock(request); - case EFileStoreRequest::ReleaseLock: - return DoReleaseLock(request); - - case EFileStoreRequest::ReadBlob: - case EFileStoreRequest::WriteBlob: - case EFileStoreRequest::GenerateBlobIds: - case EFileStoreRequest::PingSession: - case EFileStoreRequest::Ping: - continue; - default: - STORAGE_INFO( - "Uninmplemented action=" - << action << " " - << RequestName(request.GetRequestType())); - continue; - } - } - } - } - STORAGE_INFO( - "Eventlog finished n=" << EventMessageNumber - << " ptr=" << !!EventPtr); - - return MakeFuture(TCompletedRequest(true)); - } + NProto::THeaders headers); + + bool ShouldInstantProcessQueue() override; + + bool ShouldFailOnError() override; + + void Advance(); + + bool HasNextRequest() override; + + TInstant NextRequestAt() override; + + TFuture ExecuteNextRequest() override; virtual TFuture DoReadData( const NCloud::NFileStore::NProto:: diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index af2b0dca54..f54e677495 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -6,36 +6,23 @@ compare log and actual result ( S_OK E_FS_NOENT ...) read/write with multiranges (now only first processed) */ -#include "request.h" - #include "request_replay.h" -#include -#include -#include +#include +#include #include #include #include +#include #include #include #include #include -#include -#include #include -#include -#include -#include -#include #include -#include #include -#include - -#include -#include namespace NCloud::NFileStore::NLoadTest { diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 682e0a7215..ff5eba8349 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -595,7 +595,8 @@ class TLoadTest final } } }); - if (RequestGenerator->InstantProcessQueue()) { + + if (RequestGenerator->ShouldInstantProcessQueue()) { if (future.HasValue() || future.HasException()) { ProcessCompletedRequests(); } @@ -626,7 +627,7 @@ class TLoadTest final request->Action, FormatError(request->Error).c_str()); - if (RequestGenerator->FailOnError()) { + if (RequestGenerator->ShouldFailOnError()) { TestStats.Success = false; } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/ya.make b/cloud/filestore/tools/testing/loadtest/lib/ya.make index b35058ef35..32dd5d6523 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/ya.make +++ b/cloud/filestore/tools/testing/loadtest/lib/ya.make @@ -7,6 +7,7 @@ SRCS( request_data.cpp request_index.cpp request_replay_fs.cpp + request_replay.cpp test.cpp ) From 65e298dc4c095195e27fff742907c77252ae0d96 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 4 Oct 2024 15:03:13 +0000 Subject: [PATCH 27/44] better --- .../testing/loadtest/lib/request_replay.cpp | 45 ++++++++++++------- .../testing/loadtest/lib/request_replay.h | 3 +- .../testing/loadtest/protos/loadtest.proto | 17 +++---- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 5a57bfd7e4..47a37dd6c7 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -17,7 +17,7 @@ IReplayRequestGenerator::IReplayRequestGenerator( TString filesystemId, NProto::THeaders headers) : Spec(std::move(spec)) - , FileSystemId(std::move(filesystemId)) + , FileSystemIdRequest(std::move(filesystemId)) , Headers(std::move(headers)) , Session(std::move(session)) { @@ -54,23 +54,33 @@ bool IReplayRequestGenerator::ShouldFailOnError() void IReplayRequestGenerator::Advance() { - EventPtr = CurrentEvent->Next(); - if (!EventPtr) { - return; - } + for (EventPtr = CurrentEvent->Next(); EventPtr; + EventPtr = CurrentEvent->Next()) + { + MessagePtr = dynamic_cast( + EventPtr->GetProto()); - MessagePtr = - dynamic_cast(EventPtr->GetProto()); + if (!MessagePtr) { + return; + } - if (!MessagePtr) { - return; - } + const auto FileSystemId = TString{MessagePtr->GetFileSystemId()}; + if (!Spec.GetFileSystemIdFilter().empty() && + FileSystemId != Spec.GetFileSystemIdFilter()) + { + STORAGE_DEBUG("Skipped event with FileSystemId=" << FileSystemId); + continue; + } - if (FileSystemId.empty()) { - FileSystemId = TString{MessagePtr->GetFileSystemId()}; - } + if (FileSystemIdRequest.empty() && !FileSystemId.empty()) { + FileSystemIdRequest = FileSystemId; + STORAGE_INFO( + "Using FileSystemId from profile log " << FileSystemIdRequest); + } - EventMessageNumber = MessagePtr->GetRequests().size(); + EventMessageNumber = MessagePtr->GetRequests().size(); + return; + } } NThreading::TFuture @@ -87,14 +97,15 @@ IReplayRequestGenerator::ExecuteNextRequest() for (; EventMessageNumber > 0;) { auto request = MessagePtr->GetRequests()[--EventMessageNumber]; - { auto timediff = (request.GetTimestampMcs() - TimestampMcs) * Spec.GetTimeScale(); TimestampMcs = request.GetTimestampMcs(); + if (timediff > 1000000) { timediff = 0; } + const auto current = TInstant::Now(); auto diff = current - Started; @@ -107,6 +118,7 @@ IReplayRequestGenerator::ExecuteNextRequest() Sleep(slp); } + Started = current; } @@ -139,7 +151,6 @@ IReplayRequestGenerator::ExecuteNextRequest() return DoAccessNode(request); case EFileStoreRequest::ListNodes: return DoListNodes(request); - case EFileStoreRequest::AcquireLock: return DoAcquireLock(request); case EFileStoreRequest::ReleaseLock: @@ -151,6 +162,7 @@ IReplayRequestGenerator::ExecuteNextRequest() case EFileStoreRequest::PingSession: case EFileStoreRequest::Ping: continue; + default: STORAGE_INFO( "Uninmplemented action=" @@ -161,6 +173,7 @@ IReplayRequestGenerator::ExecuteNextRequest() } } } + STORAGE_INFO( "Profile log finished n=" << EventMessageNumber << " ptr=" << !!EventPtr); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index dd55e19cf9..d3637eb06e 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -19,7 +19,8 @@ class IReplayRequestGenerator: public IRequestGenerator protected: const ::NCloud::NFileStore::NProto::TReplaySpec Spec; TLog Log; - TString FileSystemId; + TString FileSystemIdRequest; // Only for GRPC + TString FileSystemIdFilter; const ::NCloud::NFileStore::NProto::THeaders Headers; NClient::ISessionPtr Session; diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index ee2f953677..ea7d703b3d 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -59,14 +59,15 @@ message TDataLoadSpec message TReplaySpec { string FileName = 1; - string ReplayRoot = 8; // not used in grpc - bool NoRead = 9; - bool NoWrite = 10; - bool Prepare = 11; // Only write files needed for reading - double TimeScale = 12; - bool WriteRandom = 13; - bool WriteEmpty = 14; - bool CreateOnRead = 15; + string ReplayRoot = 2; // not used in grpc + string FileSystemIdFilter = 3; + bool NoRead = 4; + bool NoWrite = 5; + bool Prepare = 6; // Only write files needed for reading + double TimeScale = 7; + bool WriteRandom = 8; + bool WriteEmpty = 9; + bool CreateOnRead = 10; } message TMigrationSpec From 1e27a1e004da6bbfeaaacf2569d2b293d7f50efd Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 9 Oct 2024 16:00:59 +0000 Subject: [PATCH 28/44] Some fixes --- .../filestore/tools/testing/loadtest/lib/request.h | 7 +++---- .../tools/testing/loadtest/lib/request_data.cpp | 5 ----- .../tools/testing/loadtest/lib/request_index.cpp | 5 ----- .../tools/testing/loadtest/lib/request_replay.cpp | 14 +++++++------- .../tools/testing/loadtest/lib/request_replay.h | 4 +--- .../filestore/tools/testing/loadtest/lib/test.cpp | 2 +- 6 files changed, 12 insertions(+), 25 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 96ecda3b76..b78c6f8b0a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -24,7 +24,7 @@ struct TCompletedRequest NProto::EAction Action{}; TDuration Elapsed; NProto::TError Error; - bool Stop{}; + bool Stop = false; TCompletedRequest() = default; @@ -35,7 +35,7 @@ struct TCompletedRequest {} TCompletedRequest(bool stop) noexcept - : Stop{stop} + : Stop(stop) {} }; @@ -46,10 +46,9 @@ struct IRequestGenerator virtual ~IRequestGenerator() = default; virtual bool HasNextRequest() = 0; - virtual TInstant NextRequestAt() = 0; virtual NThreading::TFuture ExecuteNextRequest() = 0; - virtual bool ShouldInstantProcessQueue() + virtual bool ShouldImmediatelyProcessQueue() { return false; } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp index 0caadd8d5f..64236a984d 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_data.cpp @@ -201,11 +201,6 @@ class TDataRequestGenerator final return true; } - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - NThreading::TFuture ExecuteNextRequest() override { const auto& action = PeekNextAction(); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp index c5a85ca7e2..ecd569f0bd 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_index.cpp @@ -96,11 +96,6 @@ class TIndexRequestGenerator final return true; } - TInstant NextRequestAt() override - { - return TInstant::Max(); - } - NThreading::TFuture ExecuteNextRequest() override { const auto& action = PeekNextAction(); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 47a37dd6c7..36bbcc2dbb 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -10,6 +10,9 @@ #include namespace NCloud::NFileStore::NLoadTest { + +//////////////////////////////////////////////////////////////////////////////// + IReplayRequestGenerator::IReplayRequestGenerator( NProto::TReplaySpec spec, ILoggingServicePtr logging, @@ -25,7 +28,9 @@ IReplayRequestGenerator::IReplayRequestGenerator( NEventLog::TOptions options; options.FileName = Spec.GetFileName(); - options.SetForceStrongOrdering(true); // need this? + + // Sort eventlog items by timestamp + options.SetForceStrongOrdering(true); CurrentEvent = CreateIterator(options); } @@ -37,12 +42,7 @@ bool IReplayRequestGenerator::HasNextRequest() return !!EventPtr; } -TInstant IReplayRequestGenerator::NextRequestAt() -{ - return TInstant::Max(); -} - -bool IReplayRequestGenerator::ShouldInstantProcessQueue() +bool IReplayRequestGenerator::ShouldImmediatelyProcessQueue() { return true; } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index d3637eb06e..d950edd32f 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -41,7 +41,7 @@ class IReplayRequestGenerator: public IRequestGenerator TString filesystemId, NProto::THeaders headers); - bool ShouldInstantProcessQueue() override; + bool ShouldImmediatelyProcessQueue() override; bool ShouldFailOnError() override; @@ -49,8 +49,6 @@ class IReplayRequestGenerator: public IRequestGenerator bool HasNextRequest() override; - TInstant NextRequestAt() override; - TFuture ExecuteNextRequest() override; virtual TFuture DoReadData( diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index ff5eba8349..a419bd88ab 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -596,7 +596,7 @@ class TLoadTest final } }); - if (RequestGenerator->ShouldInstantProcessQueue()) { + if (RequestGenerator->ShouldImmediatelyProcessQueue()) { if (future.HasValue() || future.HasException()) { ProcessCompletedRequests(); } From 49292862d29c11501a49029f0cf4b4bcdfcf0f78 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 9 Oct 2024 17:34:45 +0000 Subject: [PATCH 29/44] Describe vars --- .../testing/loadtest/lib/request_replay_fs.cpp | 4 ++-- .../tools/testing/loadtest/protos/loadtest.proto | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index f54e677495..1202a19096 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -547,7 +547,7 @@ class TReplayRequestGeneratorFs final // open mode TFileHandle fh( fullName, - OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr)); + OpenAlways | RdWr); if (fh) { nodeid = TFileStat{fh}.INode; } else { @@ -614,7 +614,7 @@ class TReplayRequestGeneratorFs final // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); - if (Spec.GetNoRead()) { + if (Spec.GetNoWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_RENAME_NODE, Started, diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index ea7d703b3d..16bc154309 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -61,12 +61,24 @@ message TReplaySpec string FileName = 1; string ReplayRoot = 2; // not used in grpc string FileSystemIdFilter = 3; + + // Skip read operations: AccessNode ReadData GetNodeAttr bool NoRead = 4; + + // Skip write operations: CreateNode CreateHandle Write RenameNode UnlinkNode bool NoWrite = 5; - bool Prepare = 6; // Only write files needed for reading + + // Factor for sleeps for respect timings from log + // Example: 0: disable sleep; 0.5: try be 2x faster; 1: keep as in log; 2: 2x times slower; double TimeScale = 7; + + // Write random data bool WriteRandom = 8; + + // Write empty data bool WriteEmpty = 9; + + // Create missing files in CreateHandle requests bool CreateOnRead = 10; } From d9e9b8d88971ee62ed77dc44f3a3f545980df0b6 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 9 Oct 2024 19:04:14 +0000 Subject: [PATCH 30/44] Fixes --- cloud/filestore/tools/testing/loadtest/lib/request.h | 4 ++-- cloud/filestore/tools/testing/loadtest/lib/test.cpp | 7 ++++--- cloud/filestore/tools/testing/loadtest/lib/ya.make | 1 - 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index b78c6f8b0a..796c30e401 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -24,7 +24,7 @@ struct TCompletedRequest NProto::EAction Action{}; TDuration Elapsed; NProto::TError Error; - bool Stop = false; + bool Stopped = false; TCompletedRequest() = default; @@ -35,7 +35,7 @@ struct TCompletedRequest {} TCompletedRequest(bool stop) noexcept - : Stop(stop) + : Stopped(stop) {} }; diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index a419bd88ab..b6e3ab5cab 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -622,9 +622,10 @@ class TLoadTest final auto code = request->Error.GetCode(); if (FAILED(code)) { - STORAGE_ERROR("%s failing test %d due to: %s", + STORAGE_ERROR( + "%s failing test %s due to: %s", MakeTestTag().c_str(), - request->Action, + NProto::EAction_Name(request->Action).c_str(), FormatError(request->Error).c_str()); if (RequestGenerator->ShouldFailOnError()) { @@ -636,7 +637,7 @@ class TLoadTest final ++stats.Requests; stats.Hist.RecordValue(request->Elapsed); - if (request->Stop) { + if (request->Stopped) { TestStats.Success = false; } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/ya.make b/cloud/filestore/tools/testing/loadtest/lib/ya.make index 32dd5d6523..6508c93276 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/ya.make +++ b/cloud/filestore/tools/testing/loadtest/lib/ya.make @@ -25,7 +25,6 @@ PEERDIR( cloud/storage/core/libs/diagnostics library/cpp/aio - library/cpp/deprecated/atomic library/cpp/testing/unittest ) From a9f68184aa1612133ded1512a4f4e42b502e72a8 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 9 Oct 2024 20:03:08 +0000 Subject: [PATCH 31/44] Fixing --- .../testing/loadtest/lib/request_replay.cpp | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 36bbcc2dbb..77e8165fe5 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -110,24 +110,27 @@ IReplayRequestGenerator::ExecuteNextRequest() auto diff = current - Started; if (timediff > diff.MicroSeconds()) { - auto slp = + auto sleep = TDuration::MicroSeconds(timediff - diff.MicroSeconds()); STORAGE_DEBUG( - "sleep=" << slp << " timediff=" << timediff - << " diff=" << diff); + "sleep=%lu timediff=%f diff=%lu", + sleep.MicroSeconds(), + timediff, + diff.MicroSeconds()); - Sleep(slp); + Sleep(sleep); } Started = current; } STORAGE_DEBUG( - "Processing message " - << EventMessageNumber << " typename=" << request.GetTypeName() - << " type=" << request.GetRequestType() << " " - << " name=" << RequestName(request.GetRequestType()) - << " json=" << request.AsJSON()); + "Processing message n=%lu typename=%s type=%d name=%s json=%s", + EventMessageNumber, + request.GetTypeName().c_str(), + request.GetRequestType(), + RequestName(request.GetRequestType()).c_str(), + ToString(request.AsJSON()).c_str()); { const auto& action = request.GetRequestType(); switch (static_cast(action)) { @@ -165,9 +168,9 @@ IReplayRequestGenerator::ExecuteNextRequest() default: STORAGE_INFO( - "Uninmplemented action=" - << action << " " - << RequestName(request.GetRequestType())); + "Uninmplemented action=%u %s", + action, + RequestName(request.GetRequestType()).c_str()); continue; } } From 262932c2dd915ed42cef7d8ef8e5653b6dfed231 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 10 Oct 2024 00:44:26 +0000 Subject: [PATCH 32/44] fmt --- .../testing/loadtest/lib/request_replay.cpp | 10 +- .../loadtest/lib/request_replay_fs.cpp | 131 ++++++++++-------- 2 files changed, 82 insertions(+), 59 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 77e8165fe5..558e71265d 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -75,7 +75,8 @@ void IReplayRequestGenerator::Advance() if (FileSystemIdRequest.empty() && !FileSystemId.empty()) { FileSystemIdRequest = FileSystemId; STORAGE_INFO( - "Using FileSystemId from profile log " << FileSystemIdRequest); + "Using FileSystemId from profile log %s", + FileSystemIdRequest.c_str()); } EventMessageNumber = MessagePtr->GetRequests().size(); @@ -113,7 +114,7 @@ IReplayRequestGenerator::ExecuteNextRequest() auto sleep = TDuration::MicroSeconds(timediff - diff.MicroSeconds()); STORAGE_DEBUG( - "sleep=%lu timediff=%f diff=%lu", + "Sleep=%lu timediff=%f diff=%lu", sleep.MicroSeconds(), timediff, diff.MicroSeconds()); @@ -178,8 +179,9 @@ IReplayRequestGenerator::ExecuteNextRequest() } STORAGE_INFO( - "Profile log finished n=" << EventMessageNumber - << " ptr=" << !!EventPtr); + "Profile log finished n=%lu hasPtr=%d", + EventMessageNumber, + !!EventPtr); return MakeFuture(TCompletedRequest(true)); } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 1202a19096..bb04c22b4f 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -75,10 +75,10 @@ class TReplayRequestGeneratorFs final public: TReplayRequestGeneratorFs( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - ISessionPtr session, - TString filesystemId, + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + ISessionPtr session, + TString filesystemId, NProto::THeaders headers) : IReplayRequestGenerator( std::move(spec), @@ -108,7 +108,9 @@ class TReplayRequestGeneratorFs final } STORAGE_DEBUG( - "node not found " << id << " map size=" << NodesLogToLocal.size()); + "Node not found id=%lu map size=%zu", + id, + NodesLogToLocal.size()); return 0; } @@ -120,8 +122,9 @@ class TReplayRequestGeneratorFs final return it->second; } STORAGE_DEBUG( - "handle not found " << id - << " map size=" << HandlesLogToActual.size()); + "Handle not found id=%lu map size=%zu", + id, + HandlesLogToActual.size()); return 0; } @@ -145,8 +148,8 @@ class TReplayRequestGeneratorFs final if (!node) { STORAGE_ERROR( - "access fail: " << " no node=" - << logRequest.GetNodeInfo().GetNodeId()); + "Access fail: no node=%lu", + logRequest.GetNodeInfo().GetNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, Started, @@ -156,8 +159,10 @@ class TReplayRequestGeneratorFs final auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); int res = access(fname.c_str(), R_OK); STORAGE_DEBUG( - "access " << node << " <- " << logRequest.GetNodeInfo().GetNodeId() - << " = " << res); + "Access %lu <- %lu = %d ", + node, + logRequest.GetNodeInfo().GetNodeId(), + res); return MakeFuture( TCompletedRequest{NProto::ACTION_ACCESS_NODE, Started, {}}); } @@ -257,9 +262,9 @@ class TReplayRequestGeneratorFs final if (!parentNode) { STORAGE_ERROR( - "create handle fail :" - << logRequest.GetNodeInfo().GetHandle() << " no parent=" - << logRequest.GetNodeInfo().GetParentNodeId()); + "Create handle %lu fail: no parent=%lu", + logRequest.GetNodeInfo().GetHandle(), + logRequest.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, Started, @@ -285,12 +290,13 @@ class TReplayRequestGeneratorFs final relativePathName = parentpath + nodeName; } STORAGE_DEBUG( - "open " << relativePathName - << " handle=" << logRequest.GetNodeInfo().GetHandle() - << " flags=" << logRequest.GetNodeInfo().GetFlags() << " " - << HandleFlagsToString(logRequest.GetNodeInfo().GetFlags()) - << " mode=" << logRequest.GetNodeInfo().GetMode() - << " node=" << logRequest.GetNodeInfo().GetNodeId()); + "Open %s handle=%lu flags=%d (%s) mode=%d node=%lu", + relativePathName.c_str(), + logRequest.GetNodeInfo().GetHandle(), + logRequest.GetNodeInfo().GetFlags(), + HandleFlagsToString(logRequest.GetNodeInfo().GetFlags()).c_str(), + logRequest.GetNodeInfo().GetMode(), + logRequest.GetNodeInfo().GetNodeId()); try { TFile fileHandle( @@ -326,11 +332,14 @@ class TReplayRequestGeneratorFs final NodePath[inode] = relativePathName; } STORAGE_DEBUG( - "Open " << fh << " <- " << logRequest.GetNodeInfo().GetHandle() - << " inode=" << inode - << " known handles=" << HandlesLogToActual.size() - << " opened=" << OpenHandles.size() - << " size=" << stat.Size); + "Open %d <- %lu inode=%lu known handles=%zu opened=%zu " + "size=%lu", + fh, + logRequest.GetNodeInfo().GetHandle(), + inode, + HandlesLogToActual.size(), + OpenHandles.size(), + stat.Size); } catch (const TFileError& error) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, @@ -372,10 +381,10 @@ class TReplayRequestGeneratorFs final const auto handle = HandleIdMapped(logRequest.GetRanges(0).GetHandle()); if (!handle) { STORAGE_WARN( - "read: no handle " - << logRequest.GetRanges(0).GetHandle() - << " ranges size=" << logRequest.GetRanges().size() - << " map size=" << HandlesLogToActual.size()); + "Read: no handle %lu ranges size=%d map size=%zu", + logRequest.GetRanges(0).GetHandle(), + logRequest.GetRanges().size(), + HandlesLogToActual.size()); return MakeFuture(TCompletedRequest{ NProto::ACTION_READ, Started, @@ -383,8 +392,10 @@ class TReplayRequestGeneratorFs final } auto& fh = OpenHandles[handle]; STORAGE_DEBUG( - "Read from " << handle << " fh.len=" << fh.GetLength() - << " fh.pos=" << fh.GetPosition()); + "Read from %lu fh.len=%ld fh.pos=%ld ", + handle, + fh.GetLength(), + fh.GetPosition()); auto buffer = Acalloc(logRequest.GetRanges().cbegin()->GetBytes()); /* @@ -413,7 +424,7 @@ class TReplayRequestGeneratorFs final return TCompletedRequest( NProto::ACTION_READ, started, - MakeError(E_IO, TStringBuilder{} << "nothing read")); + MakeError(E_IO, "nothing read")); }); } @@ -474,8 +485,10 @@ class TReplayRequestGeneratorFs final auto& fh = OpenHandles[handle]; STORAGE_DEBUG( - "Write to " << handle << " fh.length=" << fh.GetLength() - << " fh.pos=" << fh.GetPosition()); + "Write to %lu fh.length=%ld fh.pos=%ld", + handle, + fh.GetLength(), + fh.GetPosition()); // TODO(proller): TEST USE AFTER FREE on buffer TFileHandle FileHandle{fh.GetHandle()}; const auto writeFuture = AsyncIO.Write( @@ -545,9 +558,7 @@ class TReplayRequestGeneratorFs final case NProto::E_REGULAR_NODE: { // TODO(proller): transform r.GetNodeInfo().GetMode() to correct // open mode - TFileHandle fh( - fullName, - OpenAlways | RdWr); + TFileHandle fh(fullName, OpenAlways | RdWr); if (fh) { nodeid = TFileStat{fh}.INode; } else { @@ -637,13 +648,20 @@ class TReplayRequestGeneratorFs final const auto renameres = NFs::Rename(fullName, newFullName); - STORAGE_DEBUG( - "rename " << fullName << " => " << newFullName << " : " << renameres - << (renameres - ? (TStringBuilder{} - << " errno=" << LastSystemError() << " err=" - << LastSystemErrorText(LastSystemError())) - : TStringBuilder{})); + if (renameres) { + STORAGE_DEBUG( + "Rename fail %s => %s : %d errno=%d err=%s", + fullName.c_str(), + newFullName.c_str(), + renameres, + LastSystemError(), + LastSystemErrorText(LastSystemError())); + } else { + STORAGE_DEBUG( + "Renamed %s => %s ", + fullName.c_str(), + newFullName.c_str()); + } return MakeFuture( TCompletedRequest{NProto::ACTION_RENAME_NODE, Started, {}}); } @@ -668,8 +686,8 @@ class TReplayRequestGeneratorFs final NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (!parentNodeId) { STORAGE_WARN( - "unlink : no parent orig=" - << logRequest.GetNodeInfo().GetParentNodeId()); + "Unlink : no parent orig=%lu", + logRequest.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest( NProto::ACTION_REMOVE_NODE, Started, @@ -679,7 +697,7 @@ class TReplayRequestGeneratorFs final PathByNode(parentNodeId) + logRequest.GetNodeInfo().GetNodeName(); const auto unlinkres = NFs::Remove(fullName); - STORAGE_DEBUG("unlink " << fullName << " = " << unlinkres); + STORAGE_DEBUG("Unlink %s : %d ", fullName.c_str(), unlinkres); // TODO(proller): // NodesLogToActual.erase(...) // NodePath.erase(...) @@ -718,11 +736,13 @@ class TReplayRequestGeneratorFs final OpenHandles.erase(handleid); HandlesLogToActual.erase(logRequest.GetNodeInfo().GetHandle()); STORAGE_DEBUG( - "Close " << handleid << " <- " - << logRequest.GetNodeInfo().GetHandle() << " pos=" << pos - << " len=" << len - << " open map size=" << OpenHandles.size() - << " map size=" << HandlesLogToActual.size()); + "Close %lu <- %lu pos=%lu len=%ld open map size=%ld map size=%zu", + handleid, + logRequest.GetNodeInfo().GetHandle(), + pos, + len, + OpenHandles.size(), + HandlesLogToActual.size()); return MakeFuture( TCompletedRequest(NProto::ACTION_DESTROY_HANDLE, Started, {})); } @@ -844,9 +864,10 @@ class TReplayRequestGeneratorFs final const auto dirs = NLowLevel::ListDirAt(dir, true); if (logRequest.GetNodeInfo().GetSize() != dirs.size()) { STORAGE_DEBUG( - "Dir size differs " - << path << " log=" << logRequest.GetNodeInfo().GetSize() - << " local=" << dirs.size()); + "Dir size differs %s log=%lu local=%zu", + path.c_str(), + logRequest.GetNodeInfo().GetSize(), + dirs.size()); } return MakeFuture( From 82ce83c8a9195792e73f9f4fe041b19ff784a1a6 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 10 Oct 2024 19:19:05 +0000 Subject: [PATCH 33/44] Some fixes --- .../testing/loadtest/lib/request_replay.cpp | 21 ++++--- .../testing/loadtest/lib/request_replay.h | 6 +- .../loadtest/lib/request_replay_fs.cpp | 60 +++++++++---------- 3 files changed, 43 insertions(+), 44 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 558e71265d..191a5d9e53 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -20,7 +20,7 @@ IReplayRequestGenerator::IReplayRequestGenerator( TString filesystemId, NProto::THeaders headers) : Spec(std::move(spec)) - , FileSystemIdRequest(std::move(filesystemId)) + , TargetFilesystemId(std::move(filesystemId)) , Headers(std::move(headers)) , Session(std::move(session)) { @@ -64,19 +64,19 @@ void IReplayRequestGenerator::Advance() return; } - const auto FileSystemId = TString{MessagePtr->GetFileSystemId()}; + const TString fileSystemId{MessagePtr->GetFileSystemId()}; if (!Spec.GetFileSystemIdFilter().empty() && - FileSystemId != Spec.GetFileSystemIdFilter()) + fileSystemId != Spec.GetFileSystemIdFilter()) { - STORAGE_DEBUG("Skipped event with FileSystemId=" << FileSystemId); + STORAGE_DEBUG("Skipped event with FileSystemId=" << fileSystemId); continue; } - if (FileSystemIdRequest.empty() && !FileSystemId.empty()) { - FileSystemIdRequest = FileSystemId; + if (TargetFilesystemId.empty() && !fileSystemId.empty()) { + TargetFilesystemId = fileSystemId; STORAGE_INFO( "Using FileSystemId from profile log %s", - FileSystemIdRequest.c_str()); + TargetFilesystemId.c_str()); } EventMessageNumber = MessagePtr->GetRequests().size(); @@ -102,8 +102,7 @@ IReplayRequestGenerator::ExecuteNextRequest() auto timediff = (request.GetTimestampMcs() - TimestampMcs) * Spec.GetTimeScale(); TimestampMcs = request.GetTimestampMcs(); - - if (timediff > 1000000) { + if (timediff > MaxSleepMcs) { timediff = 0; } @@ -126,7 +125,7 @@ IReplayRequestGenerator::ExecuteNextRequest() } STORAGE_DEBUG( - "Processing message n=%lu typename=%s type=%d name=%s json=%s", + "Processing message n=%d typename=%s type=%d name=%s json=%s", EventMessageNumber, request.GetTypeName().c_str(), request.GetRequestType(), @@ -179,7 +178,7 @@ IReplayRequestGenerator::ExecuteNextRequest() } STORAGE_INFO( - "Profile log finished n=%lu hasPtr=%d", + "Profile log finished n=%d hasPtr=%d", EventMessageNumber, !!EventPtr); diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index d950edd32f..b64a651a77 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -19,7 +19,7 @@ class IReplayRequestGenerator: public IRequestGenerator protected: const ::NCloud::NFileStore::NProto::TReplaySpec Spec; TLog Log; - TString FileSystemIdRequest; // Only for GRPC + TString TargetFilesystemId; // Only for GRPC TString FileSystemIdFilter; const ::NCloud::NFileStore::NProto::THeaders Headers; NClient::ISessionPtr Session; @@ -27,10 +27,12 @@ class IReplayRequestGenerator: public IRequestGenerator ui64 TimestampMcs{}; TInstant Started; + constexpr static auto MaxSleepMcs = 1000000; + private: THolder CurrentEvent; TConstEventPtr EventPtr; - ui64 EventMessageNumber = 0; + int EventMessageNumber = 0; const NProto::TProfileLogRecord* MessagePtr{}; public: diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index bb04c22b4f..9e93a8baf6 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -38,7 +38,7 @@ class TReplayRequestGeneratorFs final , public std::enable_shared_from_this { private: - TVector> Actions; + NAsyncIO::TAsyncIOService AsyncIO; TMutex StateLock; using THandleLog = ui64; @@ -47,31 +47,24 @@ class TReplayRequestGeneratorFs final using TNodeLog = ui64; using TNodeLocal = ui64; - std::atomic LastRequestId = 0; + // Map between log and actual local fs node id's + THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; + + // Relative to local root path of node. Dirs ends with / + THashMap NodePath{{RootNodeId, "/"}}; + // Collected info for log node id's struct TNode { TString Name; TNodeLog ParentLog = 0; }; - - struct THandle - { - TString Path; - }; - - THashMap Handles; - THashSet Locks; - THashSet StagedLocks; - - THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; - THashMap NodePath{{RootNodeId, "/"}}; THashMap KnownLogNodes; + // Map between log and actual local open handle id's THashMap HandlesLogToActual; - THashMap OpenHandles; - NAsyncIO::TAsyncIOService AsyncIO; + THashMap OpenHandles; public: TReplayRequestGeneratorFs( @@ -79,7 +72,7 @@ class TReplayRequestGeneratorFs final ILoggingServicePtr logging, ISessionPtr session, TString filesystemId, - NProto::THeaders headers) + NProto::THeaders headers) : IReplayRequestGenerator( std::move(spec), std::move(logging), @@ -453,16 +446,18 @@ class TReplayRequestGeneratorFs final } TGuard guard(StateLock); - const auto logHandle = logRequest.GetRanges(0).GetHandle(); - const auto handle = HandleIdMapped(logHandle); - if (!handle) { + const auto handleLog = logRequest.GetRanges(0).GetHandle(); + const auto handleLocal = HandleIdMapped(handleLog); + if (!handleLocal) { + // TODO(proller): Suggest filename, place in __lost__ if unknown, + // create and open file, continue to write return MakeFuture(TCompletedRequest( NProto::ACTION_WRITE, Started, MakeError( E_CANCELLED, TStringBuilder{} << "write cancelled: no handle =" - << logHandle))); // todo + << handleLog))); } const auto bytes = logRequest.GetRanges(0).GetBytes(); const auto offset = logRequest.GetRanges(0).GetOffset(); @@ -470,23 +465,23 @@ class TReplayRequestGeneratorFs final TString buffer; if (Spec.GetWriteRandom()) { - buffer = NUnitTest::RandomString(bytes, logHandle); + buffer = NUnitTest::RandomString(bytes, handleLog); } else if (Spec.GetWriteEmpty()) { buffer = TString{bytes, ' '}; } else { buffer = MakeBuffer( bytes, offset, - TStringBuilder{} << "handle=" << logHandle << " node=" + TStringBuilder{} << "handle=" << handleLog << " node=" << logRequest.GetNodeInfo().GetNodeId() << " bytes=" << bytes << " offset=" << offset); } - auto& fh = OpenHandles[handle]; + auto& fh = OpenHandles[handleLocal]; STORAGE_DEBUG( "Write to %lu fh.length=%ld fh.pos=%ld", - handle, + handleLocal, fh.GetLength(), fh.GetPosition()); // TODO(proller): TEST USE AFTER FREE on buffer @@ -507,7 +502,7 @@ class TReplayRequestGeneratorFs final return TCompletedRequest( NProto::ACTION_WRITE, started, - MakeError(E_IO, TStringBuilder{} << "nothing writed")); + MakeError(E_IO, TStringBuilder{} << "nothing written")); }); } @@ -551,7 +546,6 @@ class TReplayRequestGeneratorFs final auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + logRequest.GetNodeInfo().GetNewNodeName(); - ui64 nodeid = 0; bool isDir = false; switch (logRequest.GetNodeInfo().GetType()) { @@ -568,11 +562,13 @@ class TReplayRequestGeneratorFs final if (logRequest.GetNodeInfo().GetSize()) { fh.Reserve(logRequest.GetNodeInfo().GetSize()); } - } break; + break; + } case NProto::E_DIRECTORY_NODE: { isDir = true; nodeid = MakeDirectoryRecursive(fullName); - } break; + break; + } case NProto::E_LINK_NODE: { // {"TimestampMcs":1727703903595285,"DurationMcs":2432,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":267,"NewNodeName":"pack-ebe666445578da0c6157f4172ad581cd731742ec.idx","Mode":292,"Type":3,"NodeId":274,"Size":245792}} @@ -581,14 +577,16 @@ class TReplayRequestGeneratorFs final const auto targetFullName = Spec.GetReplayRoot() + "/" + PathByNode(targetNode); NFs::HardLink(targetFullName, fullName); - } break; + break; + } case NProto::E_SYMLINK_NODE: { const auto targetNode = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); const auto targetFullName = Spec.GetReplayRoot() + "/" + PathByNode(targetNode); NFs::SymLink(targetFullName, fullName); - } break; + break; + } case NProto::E_SOCK_NODE: return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, From 7d53ad92b4de5795be3e40cf92a01cc3978a965a Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 11 Oct 2024 16:20:16 +0000 Subject: [PATCH 34/44] fixes --- .../tools/testing/loadtest/lib/request_replay.h | 1 + .../testing/loadtest/lib/request_replay_fs.cpp | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index b64a651a77..8dd5953381 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -27,6 +27,7 @@ class IReplayRequestGenerator: public IRequestGenerator ui64 TimestampMcs{}; TInstant Started; + // Do not sleep too much if timestamps in log is broken constexpr static auto MaxSleepMcs = 1000000; private: diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 9e93a8baf6..1217c6ab77 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -66,6 +66,8 @@ class TReplayRequestGeneratorFs final THashMap OpenHandles; + const TString LostName = "__lost__"; + public: TReplayRequestGeneratorFs( NProto::TReplaySpec spec, @@ -186,7 +188,7 @@ class TReplayRequestGeneratorFs final } if (parentPath.empty()) { parentPath = - "/__lost__/" + ToString(it->second.ParentLog) + "/"; + "/" + LostName + "/" + ToString(it->second.ParentLog) + "/"; } const auto name = parentPath + @@ -292,14 +294,15 @@ class TReplayRequestGeneratorFs final logRequest.GetNodeInfo().GetNodeId()); try { + EOpenModeFlag mode{}; + if (Spec.GetCreateOnRead()) { + mode = OpenAlways; + } else { + mode = OpenExisting; + } TFile fileHandle( Spec.GetReplayRoot() + relativePathName, - // OpenAlways | (Spec.GetNoWrite() ? RdOnly : RdWr) - FileOpenFlags( - logRequest.GetNodeInfo().GetFlags(), - Spec.GetNoWrite() ? OpenExisting - : Spec.GetCreateOnRead() ? OpenAlways - : OpenExisting)); + FileOpenFlags(logRequest.GetNodeInfo().GetFlags(), mode)); if (!fileHandle.IsOpen()) { return MakeFuture(TCompletedRequest{ From 277106b39192fddc1d22596e96bd506d917aba7e Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 11 Oct 2024 16:43:46 +0000 Subject: [PATCH 35/44] fixes --- .../loadtest/lib/request_replay_fs.cpp | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 1217c6ab77..f611de4864 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -106,7 +106,7 @@ class TReplayRequestGeneratorFs final "Node not found id=%lu map size=%zu", id, NodesLogToLocal.size()); - return 0; + return InvalidNodeId; } THandleLocal HandleIdMapped(const THandleLog id) @@ -120,7 +120,7 @@ class TReplayRequestGeneratorFs final "Handle not found id=%lu map size=%zu", id, HandlesLogToActual.size()); - return 0; + return InvalidHandle; } private: @@ -141,7 +141,7 @@ class TReplayRequestGeneratorFs final const auto node = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); - if (!node) { + if (node == InvalidNodeId) { STORAGE_ERROR( "Access fail: no node=%lu", logRequest.GetNodeInfo().GetNodeId()); @@ -165,7 +165,9 @@ class TReplayRequestGeneratorFs final // Recursive, no infinity loop check TNodeLocal CreateDirIfMissingByNodeLog(TNodeLog nodeIdLog) { - if (const auto& nodeIdLocal = NodeIdMapped(nodeIdLog)) { + if (const auto& nodeIdLocal = NodeIdMapped(nodeIdLog); + nodeIdLocal != InvalidNodeId) + { return nodeIdLocal; } @@ -175,7 +177,7 @@ class TReplayRequestGeneratorFs final } auto parent = NodeIdMapped(it->second.ParentLog); - if (!parent && it->second.ParentLog && + if (parent == InvalidNodeId && it->second.ParentLog && nodeIdLog != it->second.ParentLog) { parent = CreateDirIfMissingByNodeLog(it->second.ParentLog); @@ -242,20 +244,21 @@ class TReplayRequestGeneratorFs final if (relativePathName.empty()) { auto parentNode = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!parentNode) { + if (parentNode == InvalidNodeId) { parentNode = NodeIdMapped( KnownLogNodes[logRequest.GetNodeInfo().GetParentNodeId()] .ParentLog); } - if (!parentNode && logRequest.GetNodeInfo().GetParentNodeId() != - logRequest.GetNodeInfo().GetNodeId()) + if (parentNode == InvalidNodeId && + logRequest.GetNodeInfo().GetParentNodeId() != + logRequest.GetNodeInfo().GetNodeId()) { parentNode = CreateDirIfMissingByNodeLog( logRequest.GetNodeInfo().GetParentNodeId()); } - if (!parentNode) { + if (parentNode == InvalidNodeId) { STORAGE_ERROR( "Create handle %lu fail: no parent=%lu", logRequest.GetNodeInfo().GetHandle(), @@ -375,7 +378,7 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); const auto handle = HandleIdMapped(logRequest.GetRanges(0).GetHandle()); - if (!handle) { + if (handle == InvalidHandle) { STORAGE_WARN( "Read: no handle %lu ranges size=%d map size=%zu", logRequest.GetRanges(0).GetHandle(), @@ -451,7 +454,7 @@ class TReplayRequestGeneratorFs final const auto handleLog = logRequest.GetRanges(0).GetHandle(); const auto handleLocal = HandleIdMapped(handleLog); - if (!handleLocal) { + if (handleLocal == InvalidHandle) { // TODO(proller): Suggest filename, place in __lost__ if unknown, // create and open file, continue to write return MakeFuture(TCompletedRequest( @@ -541,10 +544,7 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); - auto parentNode = - NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); - - parentNode = CreateDirIfMissingByNodeLog( + const auto parentNode = CreateDirIfMissingByNodeLog( logRequest.GetNodeInfo().GetNewParentNodeId()); auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + @@ -635,16 +635,16 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); - const auto parentnodeid = + const auto parentNodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - auto fullName = Spec.GetReplayRoot() + PathByNode(parentnodeid) + + auto fullName = Spec.GetReplayRoot() + PathByNode(parentNodeId) + logRequest.GetNodeInfo().GetNodeName(); - const auto newparentnodeid = + const auto newParentNodeId = NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); - auto newFullName = Spec.GetReplayRoot() + PathByNode(newparentnodeid) + + auto newFullName = Spec.GetReplayRoot() + PathByNode(newParentNodeId) + logRequest.GetNodeInfo().GetNewNodeName(); const auto renameres = NFs::Rename(fullName, newFullName); @@ -685,7 +685,7 @@ class TReplayRequestGeneratorFs final const auto parentNodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - if (!parentNodeId) { + if (parentNodeId == InvalidNodeId) { STORAGE_WARN( "Unlink : no parent orig=%lu", logRequest.GetNodeInfo().GetParentNodeId()); @@ -785,7 +785,7 @@ class TReplayRequestGeneratorFs final const auto nodeid = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); - if (!nodeid) { + if (nodeid == InvalidNodeId) { return MakeFuture(TCompletedRequest{ NProto::ACTION_GET_NODE_ATTR, Started, @@ -831,7 +831,7 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); const auto nodeid = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); - if (!nodeid) { + if (nodeid == InvalidNodeId) { return MakeFuture(TCompletedRequest{ NProto::ACTION_LIST_NODES, Started, From bd9b7de4d2923ee7ff84ace2b01a90034679291d Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 19:21:14 +0000 Subject: [PATCH 36/44] No stop result --- .../filestore/tools/testing/loadtest/lib/request.h | 5 ----- .../tools/testing/loadtest/lib/request_replay.cpp | 8 +++++--- cloud/filestore/tools/testing/loadtest/lib/test.cpp | 13 +++++++------ 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index 796c30e401..cd92e966f8 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -24,7 +24,6 @@ struct TCompletedRequest NProto::EAction Action{}; TDuration Elapsed; NProto::TError Error; - bool Stopped = false; TCompletedRequest() = default; @@ -33,10 +32,6 @@ struct TCompletedRequest , Elapsed(TInstant::Now() - start) , Error(std::move(error)) {} - - TCompletedRequest(bool stop) noexcept - : Stopped(stop) - {} }; //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 191a5d9e53..ab901ab248 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -68,7 +68,9 @@ void IReplayRequestGenerator::Advance() if (!Spec.GetFileSystemIdFilter().empty() && fileSystemId != Spec.GetFileSystemIdFilter()) { - STORAGE_DEBUG("Skipped event with FileSystemId=" << fileSystemId); + STORAGE_DEBUG( + "Skipped event with FileSystemId=%s", + fileSystemId.c_str()); continue; } @@ -88,7 +90,7 @@ NThreading::TFuture IReplayRequestGenerator::ExecuteNextRequest() { if (!HasNextRequest()) { - return MakeFuture(TCompletedRequest(true)); + return {}; } for (; EventPtr; Advance()) { @@ -182,7 +184,7 @@ IReplayRequestGenerator::ExecuteNextRequest() EventMessageNumber, !!EventPtr); - return MakeFuture(TCompletedRequest(true)); + return {}; } } // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index b6e3ab5cab..575a24bbd6 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -582,9 +582,14 @@ class TLoadTest final return false; } - ++CurrentIoDepth; auto self = weak_from_this(); - const auto future = RequestGenerator->ExecuteNextRequest().Apply( + const auto future = RequestGenerator->ExecuteNextRequest(); + if (!future.Initialized()) { + TestStats.Success = false; + return false; + } + ++CurrentIoDepth; + future.Apply( [=](const TFuture& future) { if (auto ptr = self.lock()) { @@ -636,10 +641,6 @@ class TLoadTest final auto& stats = TestStats.ActionStats[request->Action]; ++stats.Requests; stats.Hist.RecordValue(request->Elapsed); - - if (request->Stopped) { - TestStats.Success = false; - } } } From 369246dd9d204e386f19185248725b1528dff21f Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 19:43:31 +0000 Subject: [PATCH 37/44] comment --- cloud/filestore/tools/testing/loadtest/lib/request.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request.h b/cloud/filestore/tools/testing/loadtest/lib/request.h index cd92e966f8..c483e5d91a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request.h @@ -43,6 +43,8 @@ struct IRequestGenerator virtual bool HasNextRequest() = 0; virtual NThreading::TFuture ExecuteNextRequest() = 0; + // With false collect request futures and process them in bulk + // With true process every request future immediately after ExecuteNextRequest virtual bool ShouldImmediatelyProcessQueue() { return false; From 8fb51419fe3368da0c89a79d3d62baed8c07c39e Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 19:45:44 +0000 Subject: [PATCH 38/44] remoove TargetFilesystemId <- return me! --- .../tools/testing/loadtest/lib/request_replay.cpp | 10 +--------- .../tools/testing/loadtest/lib/request_replay.h | 1 - 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index ab901ab248..b242a40953 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -17,10 +17,9 @@ IReplayRequestGenerator::IReplayRequestGenerator( NProto::TReplaySpec spec, ILoggingServicePtr logging, NClient::ISessionPtr session, - TString filesystemId, + TString /*filesystemId*/, NProto::THeaders headers) : Spec(std::move(spec)) - , TargetFilesystemId(std::move(filesystemId)) , Headers(std::move(headers)) , Session(std::move(session)) { @@ -74,13 +73,6 @@ void IReplayRequestGenerator::Advance() continue; } - if (TargetFilesystemId.empty() && !fileSystemId.empty()) { - TargetFilesystemId = fileSystemId; - STORAGE_INFO( - "Using FileSystemId from profile log %s", - TargetFilesystemId.c_str()); - } - EventMessageNumber = MessagePtr->GetRequests().size(); return; } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index 8dd5953381..7ab65c6578 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -19,7 +19,6 @@ class IReplayRequestGenerator: public IRequestGenerator protected: const ::NCloud::NFileStore::NProto::TReplaySpec Spec; TLog Log; - TString TargetFilesystemId; // Only for GRPC TString FileSystemIdFilter; const ::NCloud::NFileStore::NProto::THeaders Headers; NClient::ISessionPtr Session; From 8a6c2bc1aeee9140ee4c64a2220d44744ffaec17 Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 19:57:31 +0000 Subject: [PATCH 39/44] clean --- .../testing/loadtest/lib/request_replay.cpp | 108 ++++++++++-------- .../testing/loadtest/lib/request_replay.h | 2 + .../testing/loadtest/protos/loadtest.proto | 2 + 3 files changed, 63 insertions(+), 49 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index b242a40953..0248d54f09 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -14,11 +14,11 @@ namespace NCloud::NFileStore::NLoadTest { //////////////////////////////////////////////////////////////////////////////// IReplayRequestGenerator::IReplayRequestGenerator( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - NClient::ISessionPtr session, - TString /*filesystemId*/, - NProto::THeaders headers) + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString /*filesystemId*/, + NProto::THeaders headers) : Spec(std::move(spec)) , Headers(std::move(headers)) , Session(std::move(session)) @@ -78,6 +78,52 @@ void IReplayRequestGenerator::Advance() } } +TFuture IReplayRequestGenerator::ProcessRequest( + const NProto::TProfileLogRequestInfo& request) +{ + const auto& action = request.GetRequestType(); + switch (static_cast(action)) { + case EFileStoreRequest::ReadData: + return DoReadData(request); + case EFileStoreRequest::WriteData: + return DoWrite(request); + case EFileStoreRequest::CreateNode: + return DoCreateNode(request); + case EFileStoreRequest::RenameNode: + return DoRenameNode(request); + case EFileStoreRequest::UnlinkNode: + return DoUnlinkNode(request); + case EFileStoreRequest::CreateHandle: + return DoCreateHandle(request); + case EFileStoreRequest::DestroyHandle: + return DoDestroyHandle(request); + case EFileStoreRequest::GetNodeAttr: + return DoGetNodeAttr(request); + case EFileStoreRequest::AccessNode: + return DoAccessNode(request); + case EFileStoreRequest::ListNodes: + return DoListNodes(request); + case EFileStoreRequest::AcquireLock: + return DoAcquireLock(request); + case EFileStoreRequest::ReleaseLock: + return DoReleaseLock(request); + + case EFileStoreRequest::ReadBlob: + case EFileStoreRequest::WriteBlob: + case EFileStoreRequest::GenerateBlobIds: + case EFileStoreRequest::PingSession: + case EFileStoreRequest::Ping: + return {}; + + default: + STORAGE_INFO( + "Uninmplemented action=%u %s", + action, + RequestName(request.GetRequestType()).c_str()); + return {}; + } +} + NThreading::TFuture IReplayRequestGenerator::ExecuteNextRequest() { @@ -91,7 +137,8 @@ IReplayRequestGenerator::ExecuteNextRequest() } for (; EventMessageNumber > 0;) { - auto request = MessagePtr->GetRequests()[--EventMessageNumber]; + NProto::TProfileLogRequestInfo request = + MessagePtr->GetRequests()[--EventMessageNumber]; { auto timediff = (request.GetTimestampMcs() - TimestampMcs) * Spec.GetTimeScale(); @@ -119,54 +166,17 @@ IReplayRequestGenerator::ExecuteNextRequest() } STORAGE_DEBUG( - "Processing message n=%d typename=%s type=%d name=%s json=%s", + "Processing message n=%d typename=%s type=%d name=%s data=%s", EventMessageNumber, request.GetTypeName().c_str(), request.GetRequestType(), RequestName(request.GetRequestType()).c_str(), - ToString(request.AsJSON()).c_str()); + request.ShortDebugString().Quote().c_str()); + + if (const auto future = ProcessRequest(request); + future.Initialized()) { - const auto& action = request.GetRequestType(); - switch (static_cast(action)) { - case EFileStoreRequest::ReadData: - return DoReadData(request); - case EFileStoreRequest::WriteData: - return DoWrite(request); - case EFileStoreRequest::CreateNode: - return DoCreateNode(request); - case EFileStoreRequest::RenameNode: - return DoRenameNode(request); - case EFileStoreRequest::UnlinkNode: - return DoUnlinkNode(request); - case EFileStoreRequest::CreateHandle: - return DoCreateHandle(request); - case EFileStoreRequest::DestroyHandle: - return DoDestroyHandle(request); - case EFileStoreRequest::GetNodeAttr: - return DoGetNodeAttr(request); - case EFileStoreRequest::AccessNode: - return DoAccessNode(request); - case EFileStoreRequest::ListNodes: - return DoListNodes(request); - case EFileStoreRequest::AcquireLock: - return DoAcquireLock(request); - case EFileStoreRequest::ReleaseLock: - return DoReleaseLock(request); - - case EFileStoreRequest::ReadBlob: - case EFileStoreRequest::WriteBlob: - case EFileStoreRequest::GenerateBlobIds: - case EFileStoreRequest::PingSession: - case EFileStoreRequest::Ping: - continue; - - default: - STORAGE_INFO( - "Uninmplemented action=%u %s", - action, - RequestName(request.GetRequestType()).c_str()); - continue; - } + return future; } } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h index 7ab65c6578..d9b9f42f7b 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.h +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.h @@ -34,6 +34,8 @@ class IReplayRequestGenerator: public IRequestGenerator TConstEventPtr EventPtr; int EventMessageNumber = 0; const NProto::TProfileLogRecord* MessagePtr{}; + TFuture ProcessRequest( + const NProto::TProfileLogRequestInfo& request); public: IReplayRequestGenerator( diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index 16bc154309..3ac866baf6 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -60,6 +60,8 @@ message TReplaySpec { string FileName = 1; string ReplayRoot = 2; // not used in grpc + + // Use profile log messages only with this FileSystemId string FileSystemIdFilter = 3; // Skip read operations: AccessNode ReadData GetNodeAttr From 6a5aa09fd7dc90f3bf7665a867fe1d2348f6c6af Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 19:57:40 +0000 Subject: [PATCH 40/44] clean --- .../tools/testing/loadtest/lib/request_replay.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 0248d54f09..1a91666bc9 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -14,11 +14,11 @@ namespace NCloud::NFileStore::NLoadTest { //////////////////////////////////////////////////////////////////////////////// IReplayRequestGenerator::IReplayRequestGenerator( - NProto::TReplaySpec spec, - ILoggingServicePtr logging, - NClient::ISessionPtr session, - TString /*filesystemId*/, - NProto::THeaders headers) + NProto::TReplaySpec spec, + ILoggingServicePtr logging, + NClient::ISessionPtr session, + TString /*filesystemId*/, + NProto::THeaders headers) : Spec(std::move(spec)) , Headers(std::move(headers)) , Session(std::move(session)) From 134d73071d35fa4585c126783b02373b624e8da7 Mon Sep 17 00:00:00 2001 From: proller Date: Sat, 12 Oct 2024 20:02:30 +0000 Subject: [PATCH 41/44] read write --- .../testing/loadtest/lib/request_replay_fs.cpp | 16 ++++++++-------- .../tools/testing/loadtest/protos/loadtest.proto | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index f611de4864..7091573d57 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -130,7 +130,7 @@ class TReplayRequestGeneratorFs final { // nfs AccessNode 0.002297s S_OK {mask=4, node_id=36} - if (Spec.GetNoRead()) { + if (Spec.GetSkipRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_ACCESS_NODE, Started, @@ -368,7 +368,7 @@ class TReplayRequestGeneratorFs final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) override { - if (Spec.GetNoRead()) { + if (Spec.GetSkipRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_READ, Started, @@ -444,7 +444,7 @@ class TReplayRequestGeneratorFs final { //{"TimestampMcs":1465489895000,"DurationMcs":2790,"RequestType":44,"Ranges":[{"NodeId":2,"Handle":20680158862113389,"Offset":13,"Bytes":12}]} - if (Spec.GetNoWrite()) { + if (Spec.GetSkipWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_WRITE, Started, @@ -535,7 +535,7 @@ class TReplayRequestGeneratorFs final // nfs CreateNode 0.006404s S_OK {new_parent_node_id=1, // new_node_name=home, mode=509, node_id=12526, size=0} - if (Spec.GetNoWrite()) { + if (Spec.GetSkipWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_NODE, Started, @@ -626,7 +626,7 @@ class TReplayRequestGeneratorFs final // nfs RenameNode 0.002569s S_OK {parent_node_id=12527, // node_name=HEAD.lock, new_parent_node_id=12527, new_node_name=HEAD} // request->SetNodeId(NodesLogToActual[r.GetNodeInfo().GetNodeId()]); - if (Spec.GetNoWrite()) { + if (Spec.GetSkipWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_RENAME_NODE, Started, @@ -674,7 +674,7 @@ class TReplayRequestGeneratorFs final // UnlinkNode 0.002605s S_OK {parent_node_id=3, // node_name=tfrgYZ1} - if (Spec.GetNoWrite()) { + if (Spec.GetSkipWrite()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_REMOVE_NODE, Started, @@ -752,7 +752,7 @@ class TReplayRequestGeneratorFs final const NCloud::NFileStore::NProto::TProfileLogRequestInfo& logRequest) override { - if (Spec.GetNoRead()) { + if (Spec.GetSkipRead()) { return MakeFuture(TCompletedRequest{ NProto::ACTION_GET_NODE_ATTR, Started, @@ -841,7 +841,7 @@ class TReplayRequestGeneratorFs final << nodeid)}); } - if (!Spec.GetNoWrite()) { + if (!Spec.GetSkipWrite()) { CreateDirIfMissingByNodeLog(logRequest.GetNodeInfo().GetNodeId()); } diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index 3ac866baf6..f76853496d 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -65,10 +65,10 @@ message TReplaySpec string FileSystemIdFilter = 3; // Skip read operations: AccessNode ReadData GetNodeAttr - bool NoRead = 4; + bool SkipRead = 4; // Skip write operations: CreateNode CreateHandle Write RenameNode UnlinkNode - bool NoWrite = 5; + bool SkipWrite = 5; // Factor for sleeps for respect timings from log // Example: 0: disable sleep; 0.5: try be 2x faster; 1: keep as in log; 2: 2x times slower; From a2b4ab0be859f454269770f7c179bb878aefcf46 Mon Sep 17 00:00:00 2001 From: proller Date: Sun, 13 Oct 2024 20:03:35 +0000 Subject: [PATCH 42/44] enum --- .../testing/loadtest/lib/request_replay_fs.cpp | 5 +++-- .../testing/loadtest/protos/loadtest.proto | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 7091573d57..79f5108509 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -470,9 +470,10 @@ class TReplayRequestGeneratorFs final TString buffer; - if (Spec.GetWriteRandom()) { + if (Spec.GetWriteFill() == NProto::TReplaySpec_EWriteFill_Random) { buffer = NUnitTest::RandomString(bytes, handleLog); - } else if (Spec.GetWriteEmpty()) { + } else if (Spec.GetWriteFill() == NProto::TReplaySpec_EWriteFill_Empty) + { buffer = TString{bytes, ' '}; } else { buffer = MakeBuffer( diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index f76853496d..75a375c46d 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -74,14 +74,20 @@ message TReplaySpec // Example: 0: disable sleep; 0.5: try be 2x faster; 1: keep as in log; 2: 2x times slower; double TimeScale = 7; - // Write random data - bool WriteRandom = 8; + enum EWriteFill + { + // Write info about chunk (handle, size, offset, ...) + Info = 0; + // Write random data + Random = 1; + // Write empty data + Empty = 2; + } + + EWriteFill WriteFill = 8; - // Write empty data - bool WriteEmpty = 9; - // Create missing files in CreateHandle requests - bool CreateOnRead = 10; + bool CreateOnRead = 9; } message TMigrationSpec From 1d41d3e6251d11e95e5fdd156ab0fef72ec1f777 Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 17 Oct 2024 23:46:17 +0000 Subject: [PATCH 43/44] use fspath --- .../loadtest/lib/request_replay_fs.cpp | 78 +++++++++---------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 79f5108509..04ee466c1e 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -20,6 +20,7 @@ read/write with multiranges (now only first processed) #include #include +#include #include #include #include @@ -51,7 +52,7 @@ class TReplayRequestGeneratorFs final THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; // Relative to local root path of node. Dirs ends with / - THashMap NodePath{{RootNodeId, "/"}}; + THashMap NodePath; // Collected info for log node id's struct TNode @@ -85,6 +86,7 @@ class TReplayRequestGeneratorFs final if (Spec.GetReplayRoot().empty()) { ythrow yexception() << "ReplayRoot is not defined"; } + NodePath.emplace(RootNodeId, Spec.GetReplayRoot()); AsyncIO.Start(); } @@ -151,7 +153,7 @@ class TReplayRequestGeneratorFs final MakeError(E_FAIL, "cancelled")}); } - auto fname = Spec.GetReplayRoot() + "/" + PathByNode(node); + const auto fname = PathByNode(node); int res = access(fname.c_str(), R_OK); STORAGE_DEBUG( "Access %lu <- %lu = %d ", @@ -185,20 +187,18 @@ class TReplayRequestGeneratorFs final { auto parentPath = PathByNode(NodeIdMapped(it->second.ParentLog)); - if (parentPath.empty() && parent) { + if (parentPath.IsDefined() && parent) { parentPath = PathByNode(parent); } - if (parentPath.empty()) { - parentPath = - "/" + LostName + "/" + ToString(it->second.ParentLog) + "/"; + if (parentPath.IsDefined()) { + parentPath = PathByNode(RootNodeId) / LostName / "_nodeid_" + + ToString(it->second.ParentLog); } const auto name = - parentPath + - (it->second.Name.empty() ? "_nodeid_" + ToString(nodeIdLog) - : it->second.Name) + - "/"; - const auto nodeId = - MakeDirectoryRecursive(Spec.GetReplayRoot() + name); + parentPath / (it->second.Name.empty() + ? "_nodeid_" + ToString(nodeIdLog) + : it->second.Name); + const auto nodeId = MakeDirectoryRecursive(name); NodePath[nodeId] = name; NodesLogToLocal[nodeIdLog] = nodeId; @@ -234,14 +234,16 @@ class TReplayRequestGeneratorFs final TGuard guard(StateLock); - TString relativePathName; + TFsPath relativePathName; if (logRequest.GetNodeInfo().GetNodeId()) { - if (auto path = PathByNode(logRequest.GetNodeInfo().GetNodeId())) { + if (const auto path = + PathByNode(logRequest.GetNodeInfo().GetNodeId())) + { relativePathName = path; } } - if (relativePathName.empty()) { + if (relativePathName.IsDefined()) { auto parentNode = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); if (parentNode == InvalidNodeId) { @@ -279,13 +281,13 @@ class TReplayRequestGeneratorFs final } const auto parentpath = PathByNode(parentNode); - if (nodeName.empty() && IsDir(Spec.GetReplayRoot() + parentpath)) { + if (nodeName.empty() && parentpath.IsDirectory()) { nodeName = KnownLogNodes[logRequest.GetNodeInfo().GetParentNodeId()] .Name; } - relativePathName = parentpath + nodeName; + relativePathName = parentpath / nodeName; } STORAGE_DEBUG( "Open %s handle=%lu flags=%d (%s) mode=%d node=%lu", @@ -304,7 +306,7 @@ class TReplayRequestGeneratorFs final mode = OpenExisting; } TFile fileHandle( - Spec.GetReplayRoot() + relativePathName, + relativePathName, FileOpenFlags(logRequest.GetNodeInfo().GetFlags(), mode)); if (!fileHandle.IsOpen()) { @@ -323,8 +325,7 @@ class TReplayRequestGeneratorFs final OpenHandles[fh] = fileHandle; HandlesLogToActual[logRequest.GetNodeInfo().GetHandle()] = fh; - const auto stat = - TFileStat{Spec.GetReplayRoot() + relativePathName}; + const auto stat = TFileStat{relativePathName}; const auto inode = stat.INode; if (logRequest.GetNodeInfo().GetNodeId()) { NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = inode; @@ -513,7 +514,7 @@ class TReplayRequestGeneratorFs final }); } - TString PathByNode(TNodeLocal nodeid) + TFsPath PathByNode(TNodeLocal nodeid) { if (const auto& it = NodePath.find(nodeid); it != NodePath.end()) { return it->second; @@ -548,10 +549,9 @@ class TReplayRequestGeneratorFs final const auto parentNode = CreateDirIfMissingByNodeLog( logRequest.GetNodeInfo().GetNewParentNodeId()); - auto fullName = Spec.GetReplayRoot() + "/" + PathByNode(parentNode) + - logRequest.GetNodeInfo().GetNewNodeName(); + auto fullName = + PathByNode(parentNode) / logRequest.GetNodeInfo().GetNewNodeName(); ui64 nodeid = 0; - bool isDir = false; switch (logRequest.GetNodeInfo().GetType()) { case NProto::E_REGULAR_NODE: { // TODO(proller): transform r.GetNodeInfo().GetMode() to correct @@ -569,7 +569,6 @@ class TReplayRequestGeneratorFs final break; } case NProto::E_DIRECTORY_NODE: { - isDir = true; nodeid = MakeDirectoryRecursive(fullName); break; } @@ -578,16 +577,14 @@ class TReplayRequestGeneratorFs final const auto targetNode = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); - const auto targetFullName = - Spec.GetReplayRoot() + "/" + PathByNode(targetNode); + const auto targetFullName = PathByNode(targetNode); NFs::HardLink(targetFullName, fullName); break; } case NProto::E_SYMLINK_NODE: { const auto targetNode = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); - const auto targetFullName = - Spec.GetReplayRoot() + "/" + PathByNode(targetNode); + const auto targetFullName = PathByNode(targetNode); NFs::SymLink(targetFullName, fullName); break; } @@ -611,9 +608,8 @@ class TReplayRequestGeneratorFs final // CreateIfMissing(PathByNode()) if (nodeid) { NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = nodeid; - NodePath[nodeid] = PathByNode(parentNode) + - logRequest.GetNodeInfo().GetNewNodeName() + - (isDir ? "/" : ""); + NodePath[nodeid] = PathByNode(parentNode) / + logRequest.GetNodeInfo().GetNewNodeName(); } return MakeFuture( @@ -639,13 +635,13 @@ class TReplayRequestGeneratorFs final const auto parentNodeId = NodeIdMapped(logRequest.GetNodeInfo().GetParentNodeId()); - auto fullName = Spec.GetReplayRoot() + PathByNode(parentNodeId) + - logRequest.GetNodeInfo().GetNodeName(); + auto fullName = + PathByNode(parentNodeId) / logRequest.GetNodeInfo().GetNodeName(); const auto newParentNodeId = NodeIdMapped(logRequest.GetNodeInfo().GetNewParentNodeId()); - auto newFullName = Spec.GetReplayRoot() + PathByNode(newParentNodeId) + + auto newFullName = PathByNode(newParentNodeId) / logRequest.GetNodeInfo().GetNewNodeName(); const auto renameres = NFs::Rename(fullName, newFullName); @@ -695,9 +691,8 @@ class TReplayRequestGeneratorFs final Started, MakeError(E_CANCELLED, "cancelled"))); } - const auto fullName = Spec.GetReplayRoot() + "/" + - PathByNode(parentNodeId) + - logRequest.GetNodeInfo().GetNodeName(); + const auto fullName = + PathByNode(parentNodeId) / logRequest.GetNodeInfo().GetNodeName(); const auto unlinkres = NFs::Remove(fullName); STORAGE_DEBUG("Unlink %s : %d ", fullName.c_str(), unlinkres); // TODO(proller): @@ -797,8 +792,8 @@ class TReplayRequestGeneratorFs final << " in " << NodesLogToLocal.size())}); } - auto fname = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); - [[maybe_unused]] const auto stat = TFileStat{fname}; + auto fullname = PathByNode(nodeid); + [[maybe_unused]] const auto stat = TFileStat{fullname}; return MakeFuture( TCompletedRequest(NProto::ACTION_GET_NODE_ATTR, Started, {})); } @@ -846,7 +841,7 @@ class TReplayRequestGeneratorFs final CreateDirIfMissingByNodeLog(logRequest.GetNodeInfo().GetNodeId()); } - auto path = Spec.GetReplayRoot() + "/" + PathByNode(nodeid); + const auto path = PathByNode(nodeid); if (NFs::Exists(path)) { return MakeFuture(TCompletedRequest{ NProto::ACTION_LIST_NODES, @@ -855,7 +850,6 @@ class TReplayRequestGeneratorFs final E_NOT_FOUND, TStringBuilder{} << "Local dir not found " << path)}); } - // try { TFileHandle dir{path, RdOnly}; if (!dir.IsOpen()) { return MakeFuture(TCompletedRequest{ From 7a02c7e103a9f699376c12531435209c9287a74e Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 18 Oct 2024 01:39:26 +0000 Subject: [PATCH 44/44] fixes --- .../testing/loadtest/lib/request_replay.cpp | 5 ++- .../loadtest/lib/request_replay_fs.cpp | 32 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp index 1a91666bc9..911685bc2f 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay.cpp @@ -173,9 +173,8 @@ IReplayRequestGenerator::ExecuteNextRequest() RequestName(request.GetRequestType()).c_str(), request.ShortDebugString().Quote().c_str()); - if (const auto future = ProcessRequest(request); - future.Initialized()) - { + const auto future = ProcessRequest(request); + if (future.Initialized()) { return future; } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp index 04ee466c1e..8ae7408f7a 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/request_replay_fs.cpp @@ -1,7 +1,7 @@ /* -TODO: +TODO(#1733): create file/dir modes -create handle modes (now rw) +create handle modes (now only rw) compare log and actual result ( S_OK E_FS_NOENT ...) read/write with multiranges (now only first processed) */ @@ -51,7 +51,7 @@ class TReplayRequestGeneratorFs final // Map between log and actual local fs node id's THashMap NodesLogToLocal{{RootNodeId, RootNodeId}}; - // Relative to local root path of node. Dirs ends with / + // Full path of node THashMap NodePath; // Collected info for log node id's @@ -68,6 +68,7 @@ class TReplayRequestGeneratorFs final THashMap OpenHandles; const TString LostName = "__lost__"; + const TString UnknownNodeNamePrefix = "_nodeid_"; public: TReplayRequestGeneratorFs( @@ -187,16 +188,17 @@ class TReplayRequestGeneratorFs final { auto parentPath = PathByNode(NodeIdMapped(it->second.ParentLog)); - if (parentPath.IsDefined() && parent) { + if (!parentPath.IsDefined() && parent != InvalidNodeId) { parentPath = PathByNode(parent); } - if (parentPath.IsDefined()) { - parentPath = PathByNode(RootNodeId) / LostName / "_nodeid_" + - ToString(it->second.ParentLog); + if (!parentPath.IsDefined()) { + parentPath = + PathByNode(RootNodeId) / LostName / UnknownNodeNamePrefix + + ToString(it->second.ParentLog); } const auto name = parentPath / (it->second.Name.empty() - ? "_nodeid_" + ToString(nodeIdLog) + ? UnknownNodeNamePrefix + ToString(nodeIdLog) : it->second.Name); const auto nodeId = MakeDirectoryRecursive(name); NodePath[nodeId] = name; @@ -261,14 +263,15 @@ class TReplayRequestGeneratorFs final } if (parentNode == InvalidNodeId) { - STORAGE_ERROR( - "Create handle %lu fail: no parent=%lu", - logRequest.GetNodeInfo().GetHandle(), - logRequest.GetNodeInfo().GetParentNodeId()); return MakeFuture(TCompletedRequest{ NProto::ACTION_CREATE_HANDLE, Started, - MakeError(E_FAIL, "cancelled")}); + MakeError( + E_FAIL, + Sprintf( + "Create handle %lu fail: no parent=%lu", + logRequest.GetNodeInfo().GetHandle(), + logRequest.GetNodeInfo().GetParentNodeId()))}); } auto nodeName = logRequest.GetNodeInfo().GetNodeName(); @@ -573,7 +576,7 @@ class TReplayRequestGeneratorFs final break; } case NProto::E_LINK_NODE: { - // {"TimestampMcs":1727703903595285,"DurationMcs":2432,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":267,"NewNodeName":"pack-ebe666445578da0c6157f4172ad581cd731742ec.idx","Mode":292,"Type":3,"NodeId":274,"Size":245792}} + // {"TimestampMcs":1000000000000000,"DurationMcs":2432,"RequestType":26,"ErrorCode":0,"NodeInfo":{"NewParentNodeId":267,"NewNodeName":"name.ext","Mode":292,"Type":3,"NodeId":274,"Size":245792}} const auto targetNode = NodeIdMapped(logRequest.GetNodeInfo().GetNodeId()); @@ -605,7 +608,6 @@ class TReplayRequestGeneratorFs final nodeid = TFileStat{fullName}.INode; } - // CreateIfMissing(PathByNode()) if (nodeid) { NodesLogToLocal[logRequest.GetNodeInfo().GetNodeId()] = nodeid; NodePath[nodeid] = PathByNode(parentNode) /