From bf5255f1c5674131904d869c38d46bb6fb5929f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Ludwig?= Date: Thu, 15 Feb 2024 14:30:24 +0100 Subject: [PATCH] Make the code forward compatible with std.experimental.allocator. Generalizes the code to take either IAllocator or RCIAllocator. --- http/vibe/http/client.d | 16 ++--- http/vibe/http/common.d | 39 +++++++++++- http/vibe/http/server.d | 20 +++---- inet/vibe/inet/message.d | 8 ++- stream/vibe/stream/memory.d | 21 ++++--- stream/vibe/stream/operations.d | 10 +++- utils/vibe/utils/array.d | 102 ++++++++++++++++++++++---------- utils/vibe/utils/string.d | 2 +- 8 files changed, 148 insertions(+), 70 deletions(-) diff --git a/http/vibe/http/client.d b/http/vibe/http/client.d index ac54ec612..f5ec12958 100644 --- a/http/vibe/http/client.d +++ b/http/vibe/http/client.d @@ -468,11 +468,8 @@ final class HTTPClient { private void doProxyRequest(T, U)(ref T res, U requester, ref bool close_conn, ref bool has_body) @trusted { // scope new import std.conv : to; - import vibe.container.internal.utilallocator: RegionListAllocator; - version (VibeManualMemoryManagement) - scope request_allocator = new RegionListAllocator!(shared(Mallocator), false)(1024, Mallocator.instance); - else - scope request_allocator = new RegionListAllocator!(shared(GCAllocator), true)(1024, GCAllocator.instance); + scope request_allocator = createRequestAllocator(); + scope (exit) freeRequestAllocator(request_allocator); res.dropBody(); scope(failure) @@ -542,11 +539,8 @@ final class HTTPClient { */ void request(scope void delegate(scope HTTPClientRequest req) requester, scope void delegate(scope HTTPClientResponse) responder) @trusted { // scope new - import vibe.container.internal.utilallocator: RegionListAllocator; - version (VibeManualMemoryManagement) - scope request_allocator = new RegionListAllocator!(shared(Mallocator), false)(1024, Mallocator.instance); - else - scope request_allocator = new RegionListAllocator!(shared(GCAllocator), true)(1024, GCAllocator.instance); + scope request_allocator = createRequestAllocator(); + scope (exit) freeRequestAllocator(request_allocator); scope (failure) { m_responding = false; @@ -1030,7 +1024,7 @@ final class HTTPClientResponse : HTTPResponse { m_closeConn = close_conn; } - private void initialize(bool has_body, IAllocator alloc, SysTime connected_time = Clock.currTime(UTC())) + private void initialize(Allocator)(bool has_body, Allocator alloc, SysTime connected_time = Clock.currTime(UTC())) { scope(failure) finalize(true); diff --git a/http/vibe/http/common.d b/http/vibe/http/common.d index 22e1029e0..96cbf9ef9 100644 --- a/http/vibe/http/common.d +++ b/http/vibe/http/common.d @@ -489,7 +489,7 @@ final class ChunkedOutputStream : OutputStream { } /// private - this(InterfaceProxy!OutputStream stream, IAllocator alloc, bool dummy) + this(Allocator)(InterfaceProxy!OutputStream stream, Allocator alloc, bool dummy) { m_out = stream; m_buffer = AllocAppender!(ubyte[])(alloc); @@ -611,13 +611,23 @@ final class ChunkedOutputStream : OutputStream { } /// Creates a new `ChunkedInputStream` instance. -ChunkedOutputStream createChunkedOutputStream(OS)(OS destination_stream, IAllocator allocator = theAllocator()) if (isOutputStream!OS) +ChunkedOutputStream createChunkedOutputStream(OS)(OS destination_stream) if (isOutputStream!OS) +{ + return createChunkedOutputStream(destination_stream, theAllocator()); +} +/// ditto +ChunkedOutputStream createChunkedOutputStream(OS, Allocator)(OS destination_stream, Allocator allocator) if (isOutputStream!OS) { return new ChunkedOutputStream(interfaceProxy!OutputStream(destination_stream), allocator, true); } /// Creates a new `ChunkedOutputStream` instance. -FreeListRef!ChunkedOutputStream createChunkedOutputStreamFL(OS)(OS destination_stream, IAllocator allocator = theAllocator()) if (isOutputStream!OS) +FreeListRef!ChunkedOutputStream createChunkedOutputStreamFL(OS)(OS destination_stream) if (isOutputStream!OS) +{ + return createChunkedOutputStreamFL(destination_stream, theAllocator()); +} +/// ditto +FreeListRef!ChunkedOutputStream createChunkedOutputStreamFL(OS, Allocator)(OS destination_stream, Allocator allocator) if (isOutputStream!OS) { return FreeListRef!ChunkedOutputStream(interfaceProxy!OutputStream(destination_stream), allocator, true); } @@ -1069,3 +1079,26 @@ unittest { *("foo" in m) = "baz"; assert(m["foo"] == "baz"); } + + +package auto createRequestAllocator() +{ + import vibe.container.internal.utilallocator: RegionListAllocator; + + static if (is(RegionListAllocator!(shared(GCAllocator), true) == struct)) { + version (VibeManualMemoryManagement) + return allocatorObject(RegionListAllocator!(shared(Mallocator), false)(1024, Mallocator.instance)); + else + return allocatorObject(RegionListAllocator!(shared(GCAllocator), true)(1024, GCAllocator.instance)); + } else { + version (VibeManualMemoryManagement) + return new RegionListAllocator!(shared(Mallocator), false)(1024, Mallocator.instance); + else + return new RegionListAllocator!(shared(GCAllocator), true)(1024, GCAllocator.instance); + } +} + +package void freeRequestAllocator(Allocator)(ref Allocator alloc) +{ + destroy(alloc); +} diff --git a/http/vibe/http/server.d b/http/vibe/http/server.d index e0453bd52..82afd9923 100644 --- a/http/vibe/http/server.d +++ b/http/vibe/http/server.d @@ -243,12 +243,8 @@ void handleHTTPConnection(TCPConnection connection, HTTPServerContext context) } () @trusted { - import vibe.container.internal.utilallocator: RegionListAllocator; - - version (VibeManualMemoryManagement) - scope request_allocator = new RegionListAllocator!(shared(Mallocator), false)(1024, Mallocator.instance); - else - scope request_allocator = new RegionListAllocator!(shared(GCAllocator), true)(1024, GCAllocator.instance); + scope request_allocator = createRequestAllocator(); + scope (exit) freeRequestAllocator(request_allocator); handleRequest(http_stream, connection, context, settings, keep_alive, request_allocator); } (); @@ -1183,11 +1179,13 @@ final class HTTPServerRequest : HTTPRequest { Represents a HTTP response as sent from the server side. */ final class HTTPServerResponse : HTTPResponse { + alias Allocator = typeof(vibeThreadAllocator()); + private { InterfaceProxy!Stream m_conn; InterfaceProxy!ConnectionStream m_rawConnection; InterfaceProxy!OutputStream m_bodyWriter; - IAllocator m_requestAlloc; + Allocator m_requestAlloc; FreeListRef!ChunkedOutputStream m_chunkedBodyWriter; FreeListRef!CountingOutputStream m_countingWriter; FreeListRef!ZlibOutputStream m_zlibOutputStream; @@ -1201,13 +1199,13 @@ final class HTTPServerResponse : HTTPResponse { } static if (!is(Stream == InterfaceProxy!Stream)) { - this(Stream conn, ConnectionStream raw_connection, HTTPServerSettings settings, IAllocator req_alloc) + this(Stream conn, ConnectionStream raw_connection, HTTPServerSettings settings, Allocator req_alloc) @safe scope { this(InterfaceProxy!Stream(conn), InterfaceProxy!ConnectionStream(raw_connection), settings, req_alloc); } } - this(InterfaceProxy!Stream conn, InterfaceProxy!ConnectionStream raw_connection, HTTPServerSettings settings, IAllocator req_alloc) + this(InterfaceProxy!Stream conn, InterfaceProxy!ConnectionStream raw_connection, HTTPServerSettings settings, Allocator req_alloc) @safe scope { m_conn = conn; m_rawConnection = raw_connection; @@ -2105,7 +2103,7 @@ private HTTPListener listenHTTPPlain(HTTPServerSettings settings, HTTPServerRequ private alias TLSStreamType = ReturnType!(createTLSStreamFL!(InterfaceProxy!Stream)); -private bool handleRequest(InterfaceProxy!Stream http_stream, TCPConnection tcp_connection, HTTPServerContext listen_info, ref HTTPServerSettings settings, ref bool keep_alive, scope IAllocator request_allocator) +private bool handleRequest(Allocator)(InterfaceProxy!Stream http_stream, TCPConnection tcp_connection, HTTPServerContext listen_info, ref HTTPServerSettings settings, ref bool keep_alive, scope Allocator request_allocator) @safe { import std.algorithm.searching : canFind; @@ -2375,7 +2373,7 @@ private bool handleRequest(InterfaceProxy!Stream http_stream, TCPConnection tcp_ } -private void parseRequestHeader(InputStream)(HTTPServerRequest req, InputStream http_stream, IAllocator alloc, ulong max_header_size, size_t max_header_line_size) +private void parseRequestHeader(InputStream, Allocator)(HTTPServerRequest req, InputStream http_stream, Allocator alloc, ulong max_header_size, size_t max_header_line_size) if (isInputStream!InputStream) { auto stream = FreeListRef!LimitedHTTPInputStream(http_stream, max_header_size); diff --git a/inet/vibe/inet/message.d b/inet/vibe/inet/message.d index 842d3a5d2..5df70cacf 100644 --- a/inet/vibe/inet/message.d +++ b/inet/vibe/inet/message.d @@ -32,7 +32,13 @@ import std.string; alloc = Custom allocator to use for allocating strings rfc822_compatible = Flag indicating that duplicate fields should be merged using a comma */ -void parseRFC5322Header(InputStream)(InputStream input, ref InetHeaderMap dst, size_t max_line_length = 1000, IAllocator alloc = vibeThreadAllocator(), bool rfc822_compatible = true) +void parseRFC5322Header(InputStream)(InputStream input, ref InetHeaderMap dst, size_t max_line_length = 1000) + if (isInputStream!InputStream) +{ + parseRFC5322Header(input, dst, max_line_length, vibeThreadAllocator()); +} +/// ditto +void parseRFC5322Header(InputStream, Allocator)(InputStream input, ref InetHeaderMap dst, size_t max_line_length, Allocator alloc, bool rfc822_compatible = true) if (isInputStream!InputStream) { string hdr, hdrvalue; diff --git a/stream/vibe/stream/memory.d b/stream/vibe/stream/memory.d index 4f4463ab2..47a02f837 100644 --- a/stream/vibe/stream/memory.d +++ b/stream/vibe/stream/memory.d @@ -16,9 +16,9 @@ import std.array; import std.exception; import std.typecons; -MemoryOutputStream createMemoryOutputStream(IAllocator alloc = vibeThreadAllocator()) +MemoryOutputStream createMemoryOutputStream(Allocator)(Allocator alloc = vibeThreadAllocator()) @safe nothrow { - return new MemoryOutputStream(alloc, true); + return new MemoryOutputStream(alloc, MemoryOutputStream.Dummy.init); } /** Creates a new stream with the given data array as its contents. @@ -39,23 +39,26 @@ MemoryStream createMemoryStream(ubyte[] data, bool writable = true, size_t initi */ final class MemoryOutputStream : OutputStream { @safe: + private struct Dummy {} private { AllocAppender!(ubyte[]) m_destination; } - deprecated("Use createMemoryOutputStream isntead.") - this(IAllocator alloc = vibeThreadAllocator()) - { - this(alloc, true); - } - /// private - this(IAllocator alloc, bool dummy) + this(IAllocator alloc, Dummy) nothrow { m_destination = AllocAppender!(ubyte[])(alloc); } + static if (is(RCIAllocator)) { + /// private + this(RCIAllocator alloc, Dummy) + nothrow { + m_destination = AllocAppender!(ubyte[])(alloc); + } + } + /// An array with all data written to the stream so far. @property ubyte[] data() nothrow { return m_destination.data(); } diff --git a/stream/vibe/stream/operations.d b/stream/vibe/stream/operations.d index 9982a8ece..89e8e2d3c 100644 --- a/stream/vibe/stream/operations.d +++ b/stream/vibe/stream/operations.d @@ -33,7 +33,13 @@ import core.time : Duration, seconds; An exception if either the stream end was hit without hitting a newline first, or if more than max_bytes have been read from the stream. */ -ubyte[] readLine(InputStream)(InputStream stream, size_t max_bytes = size_t.max, string linesep = "\r\n", IAllocator alloc = vibeThreadAllocator()) /*@ufcs*/ +ubyte[] readLine(InputStream)(InputStream stream, size_t max_bytes = size_t.max, string linesep = "\r\n") /*@ufcs*/ + if (isInputStream!InputStream) +{ + return readLine(stream, max_bytes, linesep, vibeThreadAllocator()); +} +/// ditto +ubyte[] readLine(InputStream, Allocator)(InputStream stream, size_t max_bytes, string linesep, Allocator alloc) /*@ufcs*/ if (isInputStream!InputStream) { auto output = AllocAppender!(ubyte[])(alloc); @@ -116,7 +122,7 @@ void readLine(R, InputStream)(InputStream stream, ref R dst, size_t max_bytes = O(n+m) in typical cases, with n being the length of the scanned input string and m the length of the marker. */ -ubyte[] readUntil(InputStream)(InputStream stream, in ubyte[] end_marker, size_t max_bytes = size_t.max, IAllocator alloc = vibeThreadAllocator()) /*@ufcs*/ +ubyte[] readUntil(InputStream, Allocator)(InputStream stream, in ubyte[] end_marker, size_t max_bytes = size_t.max, Allocator alloc = vibeThreadAllocator()) /*@ufcs*/ if (isInputStream!InputStream) { auto output = AllocAppender!(ubyte[])(alloc); diff --git a/utils/vibe/utils/array.d b/utils/vibe/utils/array.d index 9d99fa41c..a02e10251 100644 --- a/utils/vibe/utils/array.d +++ b/utils/vibe/utils/array.d @@ -30,16 +30,27 @@ struct AllocAppender(ArrayType : E[], E) { ElemType[] m_data; ElemType[] m_remaining; IAllocator m_alloc; + static if (is(RCIAllocator)) + RCIAllocator m_rcAlloc; bool m_allocatedBuffer = false; } this(IAllocator alloc, ElemType[] initial_buffer = null) - { + @safe { m_alloc = alloc; m_data = initial_buffer; m_remaining = initial_buffer; } + static if (is(RCIAllocator)) { + this(RCIAllocator alloc, ElemType[] initial_buffer = null) + @safe { + m_rcAlloc = alloc; + m_data = initial_buffer; + m_remaining = initial_buffer; + } + } + @disable this(this); @property ArrayType data() { return cast(ArrayType)m_data[0 .. m_data.length - m_remaining.length]; } @@ -47,7 +58,7 @@ struct AllocAppender(ArrayType : E[], E) { void reset(AppenderResetMode reset_mode = AppenderResetMode.keepData) { if (reset_mode == AppenderResetMode.keepData) m_data = null; - else if (reset_mode == AppenderResetMode.freeData) { if (m_allocatedBuffer) m_alloc.deallocate(m_data); m_data = null; } + else if (reset_mode == AppenderResetMode.freeData) { if (m_allocatedBuffer) withAlloc!"deallocate"(m_data); m_data = null; } m_remaining = m_data; } @@ -59,24 +70,29 @@ struct AllocAppender(ArrayType : E[], E) { */ void reserve(size_t amount) - @trusted { + @safe { size_t nelems = m_data.length - m_remaining.length; if (!m_data.length) { - m_data = cast(ElemType[])m_alloc.allocate(amount*E.sizeof); + m_data = () @trusted { return cast(ElemType[])withAlloc!"allocate"(amount*E.sizeof); } (); m_remaining = m_data; m_allocatedBuffer = true; } if (m_remaining.length < amount) { - if (m_allocatedBuffer) { - void[] vdata = m_data; - m_alloc.reallocate(vdata, (nelems+amount)*E.sizeof); - m_data = () @trusted { return cast(ElemType[])vdata; } (); - } else { - auto newdata = cast(ElemType[])m_alloc.allocate((nelems+amount)*E.sizeof); + debug { + import std.digest.crc; + auto checksum = crc32Of(cast(const(ubyte)[])m_data[0 .. nelems]); + } + if (m_allocatedBuffer) () @trusted { + auto vdata = cast(void[])m_data; + withAlloc!"reallocate"(vdata, (nelems+amount)*E.sizeof); + m_data = cast(ElemType[])vdata; + } (); else { + auto newdata = () @trusted { return cast(ElemType[])withAlloc!"allocate"((nelems+amount)*E.sizeof); } (); newdata[0 .. nelems] = m_data[0 .. nelems]; m_data = newdata; m_allocatedBuffer = true; } + debug assert(crc32Of(cast(const(ubyte)[])m_data[0 .. nelems]) == checksum); } m_remaining = m_data[nelems .. m_data.length]; } @@ -88,7 +104,7 @@ struct AllocAppender(ArrayType : E[], E) { m_remaining = m_remaining[1 .. $]; } - void put(ArrayType arr) + void put(scope ArrayType arr) @safe { if (m_remaining.length < arr.length) grow(arr.length); m_remaining[0 .. arr.length] = arr[]; @@ -96,35 +112,31 @@ struct AllocAppender(ArrayType : E[], E) { } static if( !hasAliasing!E ){ - void put(in ElemType[] arr) - @trusted - { + void put(scope const(ElemType)[] arr) @trusted { put(cast(ArrayType)arr); } } static if( is(ElemType == char) ){ void put(dchar el) - @safe - { + @trusted { if( el < 128 ) put(cast(char)el); else { char[4] buf; auto len = std.utf.encode(buf, el); - put(() @trusted { return cast(ArrayType)buf[0 .. len]; }()); + put(cast(ArrayType)buf[0 .. len]); } } } static if( is(ElemType == wchar) ){ void put(dchar el) - @safe - { + @trusted { if( el < 128 ) put(cast(wchar)el); else { wchar[3] buf; auto len = std.utf.encode(buf, el); - put(() @trusted { return cast(ArrayType)buf[0 .. len]; } ()); + put(cast(ArrayType)buf[0 .. len]); } } } @@ -144,10 +156,17 @@ struct AllocAppender(ArrayType : E[], E) { assert(n <= m_remaining.length); m_remaining = m_remaining[n .. $]; } + /// ditto + void append(scope size_t delegate(scope ElemType[] dst) del) + { + auto n = del(m_remaining); + assert(n <= m_remaining.length); + m_remaining = m_remaining[n .. $]; + } } void grow(size_t min_free) - { + @safe { if( !m_data.length && min_free < 16 ) min_free = 16; auto min_size = m_data.length + min_free - m_remaining.length; @@ -156,6 +175,15 @@ struct AllocAppender(ArrayType : E[], E) { new_size = (new_size * 3) / 2; reserve(new_size - m_data.length + m_remaining.length); } + + private auto withAlloc(string method, ARGS...)(auto ref ARGS args) + { + static if (is(RCIAllocator)) { + if (!m_rcAlloc.isNull) return __traits(getMember, m_rcAlloc, method)(args); + } + if (m_alloc) return __traits(getMember, m_alloc, method)(args); + assert(false, "Using AllocAppender with no allocator set"); + } } unittest { @@ -215,6 +243,14 @@ unittest { assert(app.data == "foo"); } +unittest { + auto app = AllocAppender!string(theAllocator); + app.reserve(3); + app.put("foo"); + app.put("bar"); + assert(app.data == "foobar"); +} + struct FixedAppender(ArrayType : E[], size_t NELEM, E) { alias ElemType = Unqual!E; @@ -574,12 +610,10 @@ deprecated unittest { struct ArraySet(Key) { - import stdx.allocator : makeArray, expandArray, dispose; - import stdx.allocator.building_blocks.affix_allocator : AffixAllocator; - private { - IAllocator AW(IAllocator a) { return a; } - alias AllocatorType = AffixAllocator!(IAllocator, int); + alias GenAllocator = typeof(vibeThreadAllocator()); + GenAllocator AW(GenAllocator a) { return a; } + alias AllocatorType = AffixAllocator!(GenAllocator, int); Key[4] m_staticEntries; Key[] m_entries; AllocatorType m_allocator; @@ -624,7 +658,7 @@ struct ArraySet(Key) return ret; } - void setAllocator(IAllocator allocator) + void setAllocator(GenAllocator allocator) in { assert(m_entries.ptr is null, "Cannot set allocator after elements have been inserted."); } do { m_allocator = AllocatorType(AW(allocator)); @@ -698,19 +732,23 @@ struct ArraySet(Key) nothrow @trusted { try { auto palloc = m_allocator._parent; - if (!palloc) { - assert(vibeThreadAllocator !is null, "No theAllocator set!?"); + if (isNull(palloc)) { + assert(!isNull(vibeThreadAllocator), "No vibeThreadAllocator set!?"); m_allocator = AllocatorType(AW(vibeThreadAllocator)); } } catch (Exception e) assert(false, e.msg); // should never throw return m_allocator; } + + private static bool isNull(GenAllocator a) + { + static if (is(RCIAllocator) && is(GenAllocator == RCIAllocator)) + return a.isNull; + else return !a; + } } @safe nothrow unittest { - import stdx.allocator : allocatorObject; - import stdx.allocator.mallocator : Mallocator; - ArraySet!int s; s.setAllocator(() @trusted { return Mallocator.instance.allocatorObject; } ()); diff --git a/utils/vibe/utils/string.d b/utils/vibe/utils/string.d index e75ce34b4..885a4a127 100644 --- a/utils/vibe/utils/string.d +++ b/utils/vibe/utils/string.d @@ -161,7 +161,7 @@ sizediff_t matchBracket(const(char)[] str, bool nested = true) } /// Same as std.string.format, just using an allocator. -string formatAlloc(ARGS...)(scope IAllocator alloc, string fmt, ARGS args) +string formatAlloc(Allocator, ARGS...)(scope Allocator alloc, string fmt, ARGS args) { auto app = AllocAppender!string(alloc); formattedWrite(() @trusted { return &app; } (), fmt, args);