diff --git a/cloud/filestore/libs/service_local/config.cpp b/cloud/filestore/libs/service_local/config.cpp index 1ea6ca65ce..7f805e8c36 100644 --- a/cloud/filestore/libs/service_local/config.cpp +++ b/cloud/filestore/libs/service_local/config.cpp @@ -2,6 +2,8 @@ #include +#include + namespace NCloud::NFileStore { namespace { @@ -18,7 +20,7 @@ namespace { xxx(MaxInodeCount, ui32, 1000000 )\ xxx(MaxHandlePerSessionCount, ui32, 10000 )\ xxx(DirectIoEnabled, bool, false )\ - xxx(DirectIoAlign, ui32, 4096 )\ + xxx(DirectIoAlign, ui32, 4_KB )\ // FILESTORE_SERVICE_CONFIG #define FILESTORE_SERVICE_DECLARE_CONFIG(name, type, value) \ diff --git a/cloud/filestore/libs/service_local/fs_data.cpp b/cloud/filestore/libs/service_local/fs_data.cpp index 073318aff9..8a8bbe39e1 100644 --- a/cloud/filestore/libs/service_local/fs_data.cpp +++ b/cloud/filestore/libs/service_local/fs_data.cpp @@ -102,7 +102,7 @@ TFuture TLocalFileSystem::ReadDataAsync( auto b = std::make_shared(request.GetLength(), align); NSan::Unpoison(b->Begin(), b->Size()); - TArrayRef data((char*)b->Begin(), (char*)b->End()); + TArrayRef data(b->Begin(), b->End()); auto promise = NewPromise(); FileIOService->AsyncRead(*handle, request.GetOffset(), data).Subscribe( [b = std::move(b), promise] (const TFuture& f) mutable { @@ -110,7 +110,8 @@ TFuture TLocalFileSystem::ReadDataAsync( try { auto bytesRead = f.GetValue(); b->TrimSize(bytesRead); - response.SetBuffer(std::move(b->GetBuffer())); + response.SetBufferOffset(b->AlignedDataOffset()); + response.SetBuffer(std::move(b->AccessBuffer())); } catch (const TServiceError& e) { *response.MutableError() = MakeError(MAKE_FILESTORE_ERROR( ErrnoToFileStoreError(STATUS_FROM_CODE(e.GetCode())))); @@ -135,10 +136,9 @@ TFuture TLocalFileSystem::WriteDataAsync( TErrorResponse(ErrorInvalidHandle(request.GetHandle()))); } - auto align = Config->GetDirectIoEnabled() ? Config->GetDirectIoAlign() : 0; - - auto b = std::make_shared(std::move(*request.MutableBuffer()), align); - TArrayRef data((char*)b->Begin(), (char*)b->End()); + auto b = std::move(*request.MutableBuffer()); + auto offset = request.GetBufferOffset(); + TArrayRef data(b.begin() + offset, b.vend()); auto promise = NewPromise(); FileIOService->AsyncWrite(*handle, request.GetOffset(), data).Subscribe( [b = std::move(b), promise] (const TFuture& f) mutable { diff --git a/cloud/filestore/libs/service_local/service_ut.cpp b/cloud/filestore/libs/service_local/service_ut.cpp index 277c9879d1..f3b7c5a107 100644 --- a/cloud/filestore/libs/service_local/service_ut.cpp +++ b/cloud/filestore/libs/service_local/service_ut.cpp @@ -761,24 +761,6 @@ struct TTestBootstrap #undef FILESTORE_DECLARE_METHOD - NProto::TReadDataResponse - ReadDataAligned(ui64 handle, ui64 offset, ui32 len, ui32 align) - { - auto request = CreateReadDataRequest(handle, offset, len); - auto dbg = request->ShortDebugString(); - auto response = Store->ReadData(Ctx, std::move(request)).GetValueSync(); - - UNIT_ASSERT_C( - SUCCEEDED(response.GetError().GetCode()), - response.GetError().GetMessage() + "@" + dbg); - - TAlignedBuffer alignedBuffer( - std::move(*response.MutableBuffer()), - align); - response.SetBuffer(alignedBuffer.Begin(), alignedBuffer.Size()); - return response; - } - NProto::TWriteDataResponse WriteDataAligned( ui64 handle, ui64 offset, @@ -794,7 +776,8 @@ struct TTestBootstrap (void*)(alignedBuffer.Begin()), (void*)buffer.data(), buffer.size()); - request->SetBuffer(std::move(alignedBuffer.GetBuffer())); + request->SetBufferOffset(alignedBuffer.AlignedDataOffset()); + request->SetBuffer(std::move(alignedBuffer.AccessBuffer())); auto dbg = request->ShortDebugString(); auto response = @@ -822,7 +805,8 @@ struct TTestBootstrap (void*)(alignedBuffer.Begin()), (void*)buffer.data(), buffer.size()); - request->SetBuffer(std::move(alignedBuffer.GetBuffer())); + request->SetBufferOffset(alignedBuffer.AlignedDataOffset()); + request->SetBuffer(std::move(alignedBuffer.AccessBuffer())); auto dbg = request->ShortDebugString(); auto response = @@ -2020,7 +2004,8 @@ Y_UNIT_TEST_SUITE(LocalFileStore) "file", TCreateHandleArgs::CREATE | TCreateHandleArgs::DIRECT) .GetHandle(); - auto data = bootstrap.ReadData(handle, 0, 100).GetBuffer(); + auto readRsp = bootstrap.ReadData(handle, 0, 100); + auto data = readRsp.GetBuffer().substr(readRsp.GetBufferOffset()); UNIT_ASSERT_VALUES_EQUAL(data.size(), 0); data = "aaaabbbbcccccdddddeeee"; @@ -2038,26 +2023,23 @@ Y_UNIT_TEST_SUITE(LocalFileStore) data.append(directIoAlign, 'y'); bootstrap.WriteDataAligned(handle, 0, data, directIoAlign); + auto readDataWithOffset = + [&bootstrap](ui64 handle, ui64 offset, ui32 len) + { + auto rsp = bootstrap.ReadData(handle, offset, len); + return rsp.GetBuffer().substr(rsp.GetBufferOffset()); + }; + // read [0, 2*align] - auto buffer = - bootstrap.ReadDataAligned(handle, 0, data.size(), directIoAlign) - .GetBuffer(); + auto buffer = readDataWithOffset(handle, 0, data.size()); UNIT_ASSERT_VALUES_EQUAL(buffer, data); // read [0, align] - buffer = - bootstrap.ReadDataAligned(handle, 0, directIoAlign, directIoAlign) - .GetBuffer(); + buffer = readDataWithOffset(handle, 0, directIoAlign); UNIT_ASSERT_VALUES_EQUAL(buffer, data.substr(0, directIoAlign)); // read [align, align] - buffer = bootstrap - .ReadDataAligned( - handle, - directIoAlign, - directIoAlign, - directIoAlign) - .GetBuffer(); + buffer = readDataWithOffset(handle, directIoAlign, directIoAlign); UNIT_ASSERT_VALUES_EQUAL( buffer, data.substr(directIoAlign, directIoAlign)); diff --git a/cloud/filestore/libs/vfs_fuse/config.cpp b/cloud/filestore/libs/vfs_fuse/config.cpp index ca541ba149..16f22fe1a1 100644 --- a/cloud/filestore/libs/vfs_fuse/config.cpp +++ b/cloud/filestore/libs/vfs_fuse/config.cpp @@ -31,7 +31,7 @@ namespace { xxx(AsyncHandleOperationPeriod, TDuration, TDuration::MilliSeconds(50) )\ \ xxx(DirectIoEnabled, bool, false )\ - xxx(DirectIoAlign, ui32, 4096 )\ + xxx(DirectIoAlign, ui32, 4_KB )\ // FILESTORE_FUSE_CONFIG #define FILESTORE_FILESYSTEM_DECLARE_CONFIG(name, type, value) \ diff --git a/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp b/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp index 1216248cbe..ee423efcec 100644 --- a/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp +++ b/cloud/filestore/libs/vfs_fuse/fs_impl_data.cpp @@ -193,16 +193,14 @@ void TFileSystem::Read( .Subscribe([=, ptr = weak_from_this()] (const auto& future) { const auto& response = future.GetValue(); if (auto self = ptr.lock(); CheckResponse(self, *callContext, req, response)) { - auto align = Config->GetDirectIoEnabled() ? Config->GetDirectIoAlign() : 0; - auto [alignedData, size] = TAlignedBuffer::ExtractAlignedData( - response.GetBuffer(), - align); + const auto& buffer = response.GetBuffer(); + ui32 bufferOffset = response.GetBufferOffset(); self->ReplyBuf( *callContext, response.GetError(), req, - alignedData, - size); + buffer.Data() + bufferOffset, + buffer.Size() - bufferOffset); } }); } @@ -236,7 +234,8 @@ void TFileSystem::Write( auto request = StartRequest(ino); request->SetHandle(fi->fh); request->SetOffset(offset); - request->SetBuffer(std::move(alignedBuffer.GetBuffer())); + request->SetBufferOffset(alignedBuffer.AlignedDataOffset()); + request->SetBuffer(std::move(alignedBuffer.AccessBuffer())); const auto handle = fi->fh; const auto reqId = callContext->RequestId; @@ -300,7 +299,8 @@ void TFileSystem::WriteBuf( auto request = StartRequest(ino); request->SetHandle(fi->fh); request->SetOffset(offset); - request->SetBuffer(std::move(alignedBuffer.GetBuffer())); + request->SetBufferOffset(alignedBuffer.AlignedDataOffset()); + request->SetBuffer(std::move(alignedBuffer.AccessBuffer())); const auto handle = fi->fh; const auto reqId = callContext->RequestId; diff --git a/cloud/filestore/public/api/protos/data.proto b/cloud/filestore/public/api/protos/data.proto index 84a80efa58..32bb3af0ee 100644 --- a/cloud/filestore/public/api/protos/data.proto +++ b/cloud/filestore/public/api/protos/data.proto @@ -145,6 +145,9 @@ message TReadDataResponse // Buffer with bytes read. bytes Buffer = 2; + // Bytes read start at buffer offset. + uint32 BufferOffset = 3; + // Optional response headers. TResponseHeaders Headers = 1000; } @@ -171,6 +174,9 @@ message TWriteDataRequest // Buffer with bytes to write. bytes Buffer = 6; + + // Bytes to write start at buffer offset. + uint32 BufferOffset = 7; } message TWriteDataResponse diff --git a/cloud/storage/core/libs/common/aligned_buffer.cpp b/cloud/storage/core/libs/common/aligned_buffer.cpp index d9f00266f7..abd6c5579c 100644 --- a/cloud/storage/core/libs/common/aligned_buffer.cpp +++ b/cloud/storage/core/libs/common/aligned_buffer.cpp @@ -1 +1,120 @@ #include "aligned_buffer.h" + +#include + +#include + +namespace NCloud { + +//////////////////////////////////////////////////////////////////////////////// + +TStringBuf TAlignedBuffer::ExtractAlignedData( + const TString& buffer, + ui32 align) +{ + auto alignedData = buffer.begin(); + if (align) { + alignedData = AlignUp(buffer.data(), align); + if (alignedData > buffer.end()) { + ythrow TServiceError(E_ARGUMENT) + << "Extracting unaligned buffer " << (void*)buffer.begin() + << " with alignment " << align + << " with size " << buffer.size(); + } + } + + return TStringBuf(alignedData, buffer.end() - alignedData); +} + +TAlignedBuffer::TAlignedBuffer() + : AlignedData(Buffer.begin()) +{} + +TAlignedBuffer::TAlignedBuffer(TAlignedBuffer&& other) + : Buffer(std::move(other.Buffer)) + , AlignedData(std::move(other.AlignedData)) +{ + other.AlignedData = other.Buffer.begin(); +} + +TAlignedBuffer& TAlignedBuffer::operator=(TAlignedBuffer&& other) +{ + Buffer = std::move(other.Buffer); + AlignedData = std::move(other.AlignedData); + other.AlignedData = other.Buffer.begin(); + return *this; +} + +TAlignedBuffer::TAlignedBuffer(ui32 size, ui32 align) + : Buffer(TString::Uninitialized(size + align)) + , AlignedData(Buffer.begin()) +{ + if (align) { + Y_DEBUG_ABORT_UNLESS(IsPowerOf2(align)); // align should be power of 2 + AlignedData = AlignUp(Buffer.data(), align); + Buffer.resize(AlignedData + size - Buffer.begin()); + } +} + +TAlignedBuffer::TAlignedBuffer(TString&& buffer, ui32 align) + : Buffer(std::move(buffer)) + , AlignedData(Buffer.begin()) +{ + if (align) { + Y_DEBUG_ABORT_UNLESS(IsPowerOf2(align)); // align should be power of 2 + AlignedData = AlignUp(Buffer.data(), align); + if (AlignedData > Buffer.end()) { + ythrow TServiceError(E_ARGUMENT) + << "Initializing from unaligned buffer " + << (void*)Buffer.begin() + << " with alignment " << align + << " with size " << Buffer.size(); + } + } +} + +size_t TAlignedBuffer::AlignedDataOffset() const +{ + return AlignedData - Buffer.begin(); +} + +char* TAlignedBuffer::Begin() +{ + return const_cast(AlignedData); +} + +const char* TAlignedBuffer::Begin() const +{ + return AlignedData; +} + +char* TAlignedBuffer::End() +{ + return const_cast(Buffer.end()); +} + +const char* TAlignedBuffer::End() const +{ + return Buffer.end(); +} + +size_t TAlignedBuffer::Size() const +{ + return End() - Begin(); +} + +void TAlignedBuffer::TrimSize(size_t size) +{ + if (size > Size()) { + ythrow TServiceError(E_ARGUMENT) + << "Tried to trim to size " << size << " > " << Size(); + } + Buffer.resize(AlignedDataOffset() + size); +} + +TString& TAlignedBuffer::AccessBuffer() +{ + return Buffer; +} + +} // namespace NCloud diff --git a/cloud/storage/core/libs/common/aligned_buffer.h b/cloud/storage/core/libs/common/aligned_buffer.h index b4b64c119f..4d8f004c87 100644 --- a/cloud/storage/core/libs/common/aligned_buffer.h +++ b/cloud/storage/core/libs/common/aligned_buffer.h @@ -1,9 +1,6 @@ #pragma once -#include - #include -#include namespace NCloud { @@ -16,105 +13,31 @@ class TAlignedBuffer const char* AlignedData; public: - static std::pair ExtractAlignedData(const TString& buffer, ui32 align) - { - auto alignedData = buffer.begin(); - if (align) { - alignedData = AlignUp(buffer.data(), align); - if (alignedData > buffer.end()) { - ythrow TServiceError(E_ARGUMENT) - << "Extracting unaligned buffer " << (void*)buffer.begin() - << " with size " << buffer.size(); - } - } - - return {alignedData, buffer.end() - alignedData}; - } - - TAlignedBuffer() - : AlignedData(Buffer.begin()) - { - } + static TStringBuf ExtractAlignedData( + const TString& buffer, + ui32 align); + TAlignedBuffer(); TAlignedBuffer(const TAlignedBuffer&) = delete; - TAlignedBuffer& operator = (const TAlignedBuffer&) = delete; - - TAlignedBuffer(TAlignedBuffer&&) = default; - TAlignedBuffer& operator = (TAlignedBuffer&&) = default; - - TAlignedBuffer(ui32 size, ui32 align) - : Buffer(TString::Uninitialized(size + align)) - , AlignedData(Buffer.begin()) - { - if (align) { - Y_ASSERT(IsPowerOf2(align)); // align should be power of 2 - AlignedData = AlignUp(Buffer.data(), align); - Buffer.resize(AlignedData + size - Buffer.begin()); - } - } + TAlignedBuffer& operator=(const TAlignedBuffer&) = delete; - TAlignedBuffer(TString&& buffer, ui32 align) - : Buffer(std::move(buffer)) - , AlignedData(Buffer.begin()) - { - if (align) { - Y_ASSERT(IsPowerOf2(align)); // align should be power of 2 - AlignedData = AlignUp(Buffer.data(), align); - if (AlignedData > Buffer.end()) { - ythrow TServiceError(E_ARGUMENT) - << "Initializing from unaligned buffer " - << (void*)Buffer.begin() - << " with size " << Buffer.size(); - } - } - } + TAlignedBuffer(TAlignedBuffer&& other); + TAlignedBuffer& operator=(TAlignedBuffer&& other); - size_t AlignedDataOffset() const - { - if (!Buffer) { - return 0; - } + TAlignedBuffer(ui32 size, ui32 align); + TAlignedBuffer(TString&& buffer, ui32 align); - return AlignedData - Buffer.begin(); - } + size_t AlignedDataOffset() const; - const char* Begin() const - { - if (!Buffer) { - return Buffer.begin(); - } + char* Begin(); + const char* Begin() const; - return AlignedData; - } + char* End(); + const char* End() const; - const char* End() const - { - if (!Buffer) { - return Buffer.end(); - } - - return Buffer.end(); - } - - size_t Size() const - { - return End() - Begin(); - } - - void TrimSize(size_t size) - { - if (size > Size()) { - ythrow TServiceError(E_ARGUMENT) - << "Tried to trim to size " << size << " > " << Size(); - } - Buffer.resize(AlignedDataOffset() + size); - } - - TString& GetBuffer() - { - return Buffer; - } + size_t Size() const; + void TrimSize(size_t size); + TString& AccessBuffer(); }; - } // namespace NCloud diff --git a/cloud/storage/core/libs/common/aligned_buffer_ut.cpp b/cloud/storage/core/libs/common/aligned_buffer_ut.cpp index 133e867726..f771efd7ea 100644 --- a/cloud/storage/core/libs/common/aligned_buffer_ut.cpp +++ b/cloud/storage/core/libs/common/aligned_buffer_ut.cpp @@ -1,5 +1,7 @@ #include "aligned_buffer.h" +#include + #include #include @@ -34,6 +36,11 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) 0, "size=" << size << " ,align=" << align << " ,buffer=" << (void*)buffer.Begin()); + UNIT_ASSERT_VALUES_EQUAL_C( + buffer.AlignedDataOffset(), + buffer.Begin() - buffer.AccessBuffer().begin(), + "size=" << size << " ,align=" << align + << " ,buffer=" << (void*)buffer.Begin()); buffers.push_back(std::move(buffer)); } } @@ -55,8 +62,8 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) auto* buffer1Mem = buffer1.Begin(); - TAlignedBuffer buffer2(std::move(buffer1.GetBuffer()), align); - UNIT_ASSERT_VALUES_EQUAL(0, buffer1.GetBuffer().size()); + TAlignedBuffer buffer2(std::move(buffer1.AccessBuffer()), align); + UNIT_ASSERT_VALUES_EQUAL(0, buffer1.AccessBuffer().size()); UNIT_ASSERT_VALUES_EQUAL(size, buffer2.Size()); auto* buffer2Mem = buffer2.Begin(); @@ -68,7 +75,6 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) ui32 align = 1 << 21; ui32 size = 5678; - TAlignedBuffer buffer0(size, align); UNIT_ASSERT_VALUES_EQUAL(buffer0.Size(), size); @@ -87,7 +93,6 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) UNIT_ASSERT_VALUES_EQUAL(buffer2.Size(), size); UNIT_ASSERT_VALUES_EQUAL(0, buffer.Size()); - TAlignedBuffer buffer3 = std::move(buffer2); UNIT_ASSERT_PTR_EQUAL(bufferMem, buffer3.Begin()); UNIT_ASSERT_VALUES_EQUAL(buffer3.Size(), size); @@ -99,7 +104,6 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) ui32 align = 1 << 21; ui32 size = 5678; - TAlignedBuffer buffer(size, align); UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), size); @@ -119,14 +123,13 @@ Y_UNIT_TEST_SUITE(TAlignedBufferTest) ui32 align = 1 << 21; ui32 size = 5678; - TAlignedBuffer buffer(size, align); UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), size); - auto [extractedAlignedData, extractedSize] = - TAlignedBuffer::ExtractAlignedData(buffer.GetBuffer(), align); - UNIT_ASSERT_PTR_EQUAL(buffer.Begin(), extractedAlignedData); - UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), extractedSize); + auto extractedBuf = + TAlignedBuffer::ExtractAlignedData(buffer.AccessBuffer(), align); + UNIT_ASSERT_PTR_EQUAL(buffer.Begin(), extractedBuf.Data()); + UNIT_ASSERT_VALUES_EQUAL(buffer.Size(), extractedBuf.Size()); TString buffer1 = "abcdefg";