From 47e86537da40d6bcd89f1de88eeccffbd5903574 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:01:02 +0200 Subject: [PATCH 1/7] updated utils, optimized parser by adding SIMD support now parser uses SIMD under x86 architecture. It also got rid of a few extra states --- go.mod | 2 +- go.sum | 2 + indigo_test.go | 15 +- initializers.go | 6 +- internal/parser/http1/requestsparser.go | 337 +++++++------------ internal/parser/http1/requestsparser_test.go | 25 +- internal/parser/http1/states.go | 6 - internal/server/http/http_bench_test.go | 8 +- 8 files changed, 134 insertions(+), 267 deletions(-) diff --git a/go.mod b/go.mod index 51af6662..8b0dc5f9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/dchest/uniuri v1.2.0 - github.com/indigo-web/utils v0.1.1 + github.com/indigo-web/utils v0.1.3 github.com/stretchr/testify v1.8.4 ) diff --git a/go.sum b/go.sum index b0037660..5f56bb74 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/dchest/uniuri v1.2.0/go.mod h1:fSzm4SLHzNZvWLvWJew423PhAzkpNQYq+uNLq4 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/indigo-web/utils v0.1.1 h1:6368QFWfenf/9kqZxDF8oWCGGiUmcxJTNSTjra9GS/c= github.com/indigo-web/utils v0.1.1/go.mod h1:fBdCfyNyprkgC0FvqaLE1B43CZgByQyhjDRxz4pNt8U= +github.com/indigo-web/utils v0.1.3 h1:eLlNmB+J+OJOX12TxhhIXKohvBUT+knr9cxpCVrwbS0= +github.com/indigo-web/utils v0.1.3/go.mod h1:fBdCfyNyprkgC0FvqaLE1B43CZgByQyhjDRxz4pNt8U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= diff --git a/indigo_test.go b/indigo_test.go index f43802a9..bbff8dae 100644 --- a/indigo_test.go +++ b/indigo_test.go @@ -80,7 +80,6 @@ func getStaticRouter(t *testing.T) router.Router { require.Equal(t, method.GET, request.Method) _, err := request.Path.Query.Get("some non-existing query key") require.Error(t, err) - require.Empty(t, request.Path.Fragment) require.Equal(t, proto.HTTP11, request.Proto) return http.RespondTo(request) @@ -91,7 +90,7 @@ func getStaticRouter(t *testing.T) router.Router { }) r.Get("/get-read-body", func(request *http.Request) http.Response { - require.Contains(t, request.Headers.Unwrap(), testHeaderKey) + require.True(t, request.Headers.Has(testHeaderKey)) requestHeader := strings.Join(request.Headers.Values(testHeaderKey), ",") require.Equal(t, testHeaderValue, requestHeader) @@ -370,15 +369,13 @@ func TestServer_Static(t *testing.T) { // are empty strings. Remove them headerLines = headerLines[:len(headerLines)-2] wantHeaderLines := []string{ - "hello: World!", - "host: " + addr, - "user-agent: Go-http-client/1.1", - "accept-encoding: gzip", - "content-length: 0", + "Hello: World!", + "Host: " + addr, + "User-Agent: Go-http-client/1.1", + "Accept-Encoding: gzip", + "Content-Length: 0", } - require.Equal(t, len(wantHeaderLines), len(headerLines)) - for _, line := range headerLines { require.True(t, contains(wantHeaderLines, line), "unwanted header line: "+line) } diff --git a/initializers.go b/initializers.go index 840f7046..bf2b09df 100644 --- a/initializers.go +++ b/initializers.go @@ -21,12 +21,12 @@ func newClient(tcpSettings settings.TCP, conn net.Conn) tcp.Client { return tcp.NewClient(conn, tcpSettings.ReadTimeout, readBuff) } -func newKeyValueArenas(s settings.Headers) (*arena.Arena, *arena.Arena) { - keyArena := arena.NewArena( +func newKeyValueArenas(s settings.Headers) (*arena.Arena[byte], *arena.Arena[byte]) { + keyArena := arena.NewArena[byte]( s.MaxKeyLength*s.Number.Default, s.MaxKeyLength*s.Number.Maximal, ) - valArena := arena.NewArena( + valArena := arena.NewArena[byte]( s.ValueSpace.Default, s.ValueSpace.Maximal, ) diff --git a/internal/parser/http1/requestsparser.go b/internal/parser/http1/requestsparser.go index deba3424..83882191 100644 --- a/internal/parser/http1/requestsparser.go +++ b/internal/parser/http1/requestsparser.go @@ -1,7 +1,9 @@ package http1 import ( + "bytes" "fmt" + "github.com/indigo-web/indigo/internal/strcomp" "strings" "github.com/indigo-web/indigo/http" @@ -29,8 +31,8 @@ type httpRequestsParser struct { headerKey string startLineBuff []byte headersValuesPool pool.ObjectPool[[]string] - headerKeyArena arena.Arena - headerValueArena arena.Arena + headerKeyArena arena.Arena[byte] + headerValueArena arena.Arena[byte] headersSettings settings.Headers begin int pointer int @@ -44,7 +46,7 @@ type httpRequestsParser struct { } func NewHTTPRequestsParser( - request *http.Request, keyArena, valArena arena.Arena, + request *http.Request, keyArena, valArena arena.Arena[byte], valuesPool pool.ObjectPool[[]string], startLineBuff []byte, headersSettings settings.Headers, ) parser.HTTPRequestsParser { return &httpRequestsParser{ @@ -61,7 +63,6 @@ func NewHTTPRequestsParser( } func (p *httpRequestsParser) Parse(data []byte) (state parser.RequestState, extra []byte, err error) { - var value string _ = *p.request requestHeaders := p.request.Headers @@ -82,10 +83,6 @@ func (p *httpRequestsParser) Parse(data []byte) (state parser.RequestState, extr goto queryDecode2Char case eFragment: goto fragment - case eFragmentDecode1Char: - goto fragmentDecode1Char - case eFragmentDecode2Char: - goto fragmentDecode2Char case eProto: goto proto case eH: @@ -112,22 +109,14 @@ func (p *httpRequestsParser) Parse(data []byte) (state parser.RequestState, extr goto protoCRLFCR case eHeaderKey: goto headerKey - case eHeaderColon: - goto headerColon case eContentLength: goto contentLength case eContentLengthCR: goto contentLengthCR - case eContentLengthCRLF: - goto contentLengthCRLF case eContentLengthCRLFCR: goto contentLengthCRLFCR case eHeaderValue: goto headerValue - case eHeaderValueCR: - goto headerValueCR - case eHeaderValueCRLF: - goto headerValueCRLF case eHeaderValueCRLFCR: goto headerValueCRLFCR default: @@ -324,63 +313,17 @@ queryDecode2Char: goto query fragment: - for i := range data { - switch data[i] { - case ' ': - p.request.Path.Fragment = uf.B2S(p.startLineBuff[p.begin:p.pointer]) - data = data[i+1:] - p.state = eProto - goto proto - case '%': - data = data[i+1:] - p.state = eFragmentDecode1Char - goto fragmentDecode1Char - case '\x00', '\n', '\r', '\t', '\b', '\a', '\v', '\f': - return parser.Error, nil, status.ErrBadRequest - default: - if p.pointer >= len(p.startLineBuff) { - return parser.Error, nil, status.ErrURITooLong - } - - p.startLineBuff[p.pointer] = data[i] - p.pointer++ + { + sp := bytes.IndexByte(data, ' ') + if sp == -1 { + return parser.Pending, nil, nil } - } - - return parser.Pending, nil, nil -fragmentDecode1Char: - if len(data) == 0 { - return parser.Pending, nil, nil - } - - if !isHex(data[0]) { - return parser.Error, nil, status.ErrURIDecoding - } - - p.urlEncodedChar = unHex(data[0]) << 4 - data = data[1:] - p.state = eFragmentDecode2Char - goto fragmentDecode2Char - -fragmentDecode2Char: - if len(data) == 0 { - return parser.Pending, nil, nil - } - - if !isHex(data[0]) { - return parser.Error, nil, status.ErrURIDecoding - } - if p.pointer >= len(p.startLineBuff) { - return parser.Error, nil, status.ErrURITooLong + data = data[sp+1:] + p.state = eProto + goto proto } - p.startLineBuff[p.pointer] = p.urlEncodedChar | unHex(data[0]) - p.pointer++ - data = data[1:] - p.state = eFragment - goto fragment - proto: if len(data) == 0 { return parser.Pending, nil, nil @@ -538,7 +481,6 @@ protoCRLF: default: // headers are here. I have to have a buffer for header key, and after receiving it, // get an appender from headers manager (and keep it in httpRequestsParser struct) - data[0] = data[0] | 0x20 p.state = eHeaderKey goto headerKey } @@ -555,113 +497,99 @@ protoCRLFCR: return parser.Error, nil, status.ErrBadRequest headerKey: - for i := range data { - switch data[i] { - case ':': - p.headersNumber++ + if len(data) == 0 { + return parser.Pending, nil, err + } - if p.headersNumber > p.headersSettings.Number.Maximal { - return parser.Error, nil, status.ErrTooManyHeaders - } + switch data[0] { + case '\n': + return parser.HeadersCompleted, data[1:], nil + case '\r': + data = data[1:] + p.state = eHeaderValueCRLFCR + goto headerValueCRLFCR + } - if !p.headerKeyArena.Append(data[:i]) { + { + colon := bytes.IndexByte(data, ':') + if colon == -1 { + if !p.headerKeyArena.Append(data...) { return parser.Error, nil, status.ErrHeaderFieldsTooLarge } - p.headerKey = uf.B2S(p.headerKeyArena.Finish()) - data = data[i+1:] - - if p.headerKey == "content-length" { - p.state = eContentLength - goto contentLength - } + return parser.Pending, nil, nil + } - p.state = eHeaderColon - goto headerColon - case '\r', '\n': - return parser.Error, nil, status.ErrBadRequest - default: - data[i] = data[i] | 0x20 + if !p.headerKeyArena.Append(data[:colon]...) { + return parser.Error, nil, status.ErrHeaderFieldsTooLarge } - } - if !p.headerKeyArena.Append(data) { - return parser.Error, nil, status.ErrHeaderFieldsTooLarge - } + p.headerKey = uf.B2S(p.headerKeyArena.Finish()) + data = data[colon+1:] - return parser.Pending, nil, nil + if p.headersNumber++; p.headersNumber > p.headersSettings.Number.Maximal { + return parser.Error, nil, status.ErrTooManyHeaders + } -headerColon: - for i := range data { - switch data[i] { - case '\r', '\n': - return parser.Error, nil, status.ErrBadRequest - case ' ': - default: - data = data[i:] - p.state = eHeaderValue - goto headerValue + if strcomp.EqualFold(p.headerKey, "content-length") { + p.state = eContentLength + goto contentLength } - } - return parser.Pending, nil, nil + p.state = eHeaderValue + goto headerValue + } contentLength: - for i := range data { - switch char := data[i]; char { - case ' ': - case '\r': - data = data[i+1:] - p.state = eContentLengthCR - goto contentLengthCR - case '\n': - data = data[i+1:] - p.state = eContentLengthCRLF - goto contentLengthCRLF - default: - if char < '0' || char > '9' { - return parser.Error, nil, status.ErrBadRequest - } - - p.contentLength = p.contentLength*10 + int(char-'0') + for i, char := range data { + if char == ' ' { + continue } - } - return parser.Pending, nil, nil + if char < '0' || char > '9' { + data = data[i:] + goto contentLengthEnd + } -contentLengthCR: - if len(data) == 0 { - return parser.Pending, nil, nil + p.contentLength = p.contentLength*10 + int(char-'0') } - if data[0] == '\n' { - data = data[1:] - p.state = eContentLengthCRLF - goto contentLengthCRLF - } - - return parser.Error, nil, status.ErrBadRequest - -contentLengthCRLF: - if len(data) == 0 { - return parser.Pending, nil, nil - } + return parser.Pending, nil, nil +contentLengthEnd: + // guaranteed, that data at this point contains AT LEAST 1 byte. + // The proof is, that this code is reachable ONLY if loop has reached a non-digit + // ascii symbol. In case loop has finished peacefully, as no more data left, but also no + // character found to satisfy the exit condition, this code will never be reached p.request.ContentLength = p.contentLength switch data[0] { + case ' ': case '\r': data = data[1:] - p.state = eContentLengthCRLFCR - goto contentLengthCRLFCR + p.state = eContentLengthCR + goto contentLengthCR case '\n': - return parser.HeadersCompleted, data[1:], nil - default: - data[0] = data[0] | 0x20 + data = data[1:] p.state = eHeaderKey goto headerKey + default: + return parser.Error, nil, status.ErrBadRequest } +contentLengthCR: + if len(data) == 0 { + return parser.Pending, nil, nil + } + + if data[0] != '\n' { + return parser.Error, nil, status.ErrBadRequest + } + + data = data[1:] + p.state = eHeaderKey + goto headerKey + contentLengthCRLFCR: if len(data) == 0 { return parser.Pending, nil, nil @@ -674,92 +602,49 @@ contentLengthCRLFCR: return parser.Error, nil, status.ErrBadRequest headerValue: - for i := range data { - switch data[i] { - case '\r': - if len(data[:i])+p.headerValueSize > p.headersSettings.MaxValueLength { - return parser.Error, nil, status.ErrHeaderFieldsTooLarge - } - - if !p.headerValueArena.Append(data[:i]) { - return parser.Error, nil, status.ErrHeaderFieldsTooLarge - } - - p.headerValueSize = 0 - data = data[i+1:] - p.state = eHeaderValueCR - goto headerValueCR - case '\n': - if len(data[:i])+p.headerValueSize > p.headersSettings.MaxValueLength { + { + lf := bytes.IndexByte(data, '\n') + if lf == -1 { + if !p.headerValueArena.Append(data...) { return parser.Error, nil, status.ErrHeaderFieldsTooLarge } - if !p.headerValueArena.Append(data[:i]) { + if p.headerValueArena.SegmentLength() > p.headersSettings.MaxValueLength { return parser.Error, nil, status.ErrHeaderFieldsTooLarge } - p.headerValueSize = 0 - data = data[i+1:] - p.state = eHeaderValueCRLF - goto headerValueCRLF + return parser.Pending, nil, nil } - } - - p.headerValueSize += len(data) - if p.headerValueSize >= p.headersSettings.MaxValueLength { - return parser.Error, nil, status.ErrHeaderFieldsTooLarge - } - - if !p.headerValueArena.Append(data) { - return parser.Error, nil, status.ErrHeaderFieldsTooLarge - } - - return parser.Pending, nil, nil - -headerValueCR: - if len(data) == 0 { - return parser.Pending, nil, nil - } - - if data[0] == '\n' { - data = data[1:] - p.state = eHeaderValueCRLF - goto headerValueCRLF - } - - return parser.Error, nil, status.ErrBadRequest + if !p.headerValueArena.Append(data[:lf]...) { + return parser.Error, nil, status.ErrHeaderFieldsTooLarge + } -headerValueCRLF: - if len(data) == 0 { - return parser.Pending, nil, nil - } + if p.headerValueArena.SegmentLength() > p.headersSettings.MaxValueLength { + return parser.Error, nil, status.ErrHeaderFieldsTooLarge + } - value = uf.B2S(p.headerValueArena.Finish()) - requestHeaders.Add(p.headerKey, value) + data = data[lf+1:] + value := uf.B2S(trimPrefixSpaces(p.headerValueArena.Finish())) + if value[len(value)-1] == '\r' { + value = value[:len(value)-1] + } - switch p.headerKey { - case "content-type": - p.request.ContentType = value - case "upgrade": - p.request.Upgrade = proto.ChooseUpgrade(value) - case "transfer-encoding": - te := parseTransferEncoding(value) - te.HasTrailer = p.request.TransferEncoding.HasTrailer - p.request.TransferEncoding = te - case "trailer": - p.request.TransferEncoding.HasTrailer = true - } + requestHeaders.Add(p.headerKey, value) + + switch { + case strcomp.EqualFold(p.headerKey, "content-type"): + p.request.ContentType = value + case strcomp.EqualFold(p.headerKey, "upgrade"): + p.request.Upgrade = proto.ChooseUpgrade(value) + case strcomp.EqualFold(p.headerKey, "transfer-encoding"): + te := parseTransferEncoding(value) + te.HasTrailer = p.request.TransferEncoding.HasTrailer + p.request.TransferEncoding = te + case strcomp.EqualFold(p.headerKey, "trailer"): + p.request.TransferEncoding.HasTrailer = true + } - switch data[0] { - case '\n': - return parser.HeadersCompleted, data[1:], nil - case '\r': - data = data[1:] - p.state = eHeaderValueCRLFCR - goto headerValueCRLFCR - default: - data[0] = data[0] | 0x20 p.state = eHeaderKey goto headerKey } @@ -813,3 +698,13 @@ func processTEToken(rawToken string, te headers.TransferEncoding) headers.Transf return te } + +func trimPrefixSpaces(b []byte) []byte { + for i, char := range b { + if char != ' ' { + return b[i:] + } + } + + return b[:0] +} diff --git a/internal/parser/http1/requestsparser_test.go b/internal/parser/http1/requestsparser_test.go index 45f22d57..23b67635 100644 --- a/internal/parser/http1/requestsparser_test.go +++ b/internal/parser/http1/requestsparser_test.go @@ -41,11 +41,11 @@ var ( func getParser() (httpparser.HTTPRequestsParser, *http.Request) { s := settings2.Default() - keyArena := arena.NewArena( + keyArena := arena.NewArena[byte]( s.Headers.MaxKeyLength*s.Headers.Number.Default, s.Headers.MaxKeyLength*s.Headers.Number.Maximal, ) - valArena := arena.NewArena( + valArena := arena.NewArena[byte]( s.Headers.ValueSpace.Default, s.Headers.ValueSpace.Maximal, ) objPool := pool.NewObjectPool[[]string](20) @@ -64,7 +64,6 @@ func getParser() (httpparser.HTTPRequestsParser, *http.Request) { type wantedRequest struct { Headers *headers.Headers Path string - Fragment string Method method.Method Protocol proto.Proto } @@ -72,7 +71,6 @@ type wantedRequest struct { func compareRequests(t *testing.T, wanted wantedRequest, actual *http.Request) { require.Equal(t, wanted.Method, actual.Method) require.Equal(t, wanted.Path, actual.Path.String) - require.Equal(t, wanted.Fragment, actual.Path.Fragment) require.Equal(t, wanted.Protocol, actual.Proto) hdrs := wanted.Headers.Unwrap() @@ -288,7 +286,6 @@ func TestHttpRequestsParser_Parse_GET(t *testing.T) { wanted := wantedRequest{ Method: method.GET, Path: "/", - Fragment: "Some where", Protocol: proto.HTTP11, Headers: headers.NewHeaders(nil), } @@ -308,7 +305,6 @@ func TestHttpRequestsParser_Parse_GET(t *testing.T) { wanted := wantedRequest{ Method: method.GET, Path: "/", - Fragment: "Some where", Protocol: proto.HTTP11, Headers: headers.NewHeaders(nil), } @@ -328,7 +324,6 @@ func TestHttpRequestsParser_Parse_GET(t *testing.T) { wanted := wantedRequest{ Method: method.GET, Path: "/", - Fragment: "Fragment", Protocol: proto.HTTP11, Headers: headers.NewHeaders(nil), } @@ -509,22 +504,6 @@ func TestHttpRequestsParser_Parse_Negative(t *testing.T) { require.Equal(t, httpparser.HeadersCompleted, state) }) - t.Run("HeaderWithoutColon", func(t *testing.T) { - parser, _ := getParser() - raw := []byte("GET / HTTP/1.1\r\nsome header some value\r\n\r\n") - state, _, err := parser.Parse(raw) - require.EqualError(t, err, status.ErrBadRequest.Error()) - require.Equal(t, httpparser.Error, state) - }) - - t.Run("HeaderWithoutColon", func(t *testing.T) { - parser, _ := getParser() - raw := []byte("GET / HTTP/1.1\r\nsome header some value\r\n\r\n") - state, _, err := parser.Parse(raw) - require.EqualError(t, err, status.ErrBadRequest.Error()) - require.Equal(t, httpparser.Error, state) - }) - t.Run("MajorHTTPVersionOverflow", func(t *testing.T) { parser, _ := getParser() raw := []byte("GET / HTTP/335.1\r\n\r\n") diff --git a/internal/parser/http1/states.go b/internal/parser/http1/states.go index 81f08ebf..90b1a6c5 100644 --- a/internal/parser/http1/states.go +++ b/internal/parser/http1/states.go @@ -11,8 +11,6 @@ const ( eQueryDecode1Char eQueryDecode2Char eFragment - eFragmentDecode1Char - eFragmentDecode2Char eProto eH eHT @@ -26,14 +24,10 @@ const ( eProtoCRLF eProtoCRLFCR eHeaderKey - eHeaderColon eContentLength eContentLengthCR - eContentLengthCRLF eContentLengthCRLFCR eHeaderValue - eHeaderValueCR - eHeaderValueCRLF eHeaderValueCRLFCR ) diff --git a/internal/server/http/http_bench_test.go b/internal/server/http/http_bench_test.go index 8de0949b..0b2bfb20 100644 --- a/internal/server/http/http_bench_test.go +++ b/internal/server/http/http_bench_test.go @@ -125,11 +125,11 @@ func Benchmark_Get(b *testing.B) { hdrs, q, http.NewResponse(), dummy.NewNopConn(), http.NewBody(bodyReader, decode.NewDecoder()), nil, false, ) - keyArena := arena.NewArena( + keyArena := arena.NewArena[byte]( s.Headers.MaxKeyLength*s.Headers.Number.Default, s.Headers.MaxKeyLength*s.Headers.Number.Maximal, ) - valArena := arena.NewArena( + valArena := arena.NewArena[byte]( s.Headers.ValueSpace.Default, s.Headers.ValueSpace.Maximal, ) objPool := pool.NewObjectPool[[]string](20) @@ -195,11 +195,11 @@ func Benchmark_Post(b *testing.B) { hdrs, q, http.NewResponse(), dummy.NewNopConn(), http.NewBody(reader, decode.NewDecoder()), nil, false, ) - keyArena := arena.NewArena( + keyArena := arena.NewArena[byte]( s.Headers.MaxKeyLength*s.Headers.Number.Default, s.Headers.MaxKeyLength*s.Headers.Number.Maximal, ) - valArena := arena.NewArena( + valArena := arena.NewArena[byte]( s.Headers.ValueSpace.Default, s.Headers.ValueSpace.Maximal, ) objPool := pool.NewObjectPool[[]string](20) From a949ab06d284d0645663fe294173ff5eb88d0565 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:01:27 +0200 Subject: [PATCH 2/7] added equalfold for strings comparing it's pretty faster than bytes.EqualFold() --- internal/strcomp/equalfold.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 internal/strcomp/equalfold.go diff --git a/internal/strcomp/equalfold.go b/internal/strcomp/equalfold.go new file mode 100644 index 00000000..832a3922 --- /dev/null +++ b/internal/strcomp/equalfold.go @@ -0,0 +1,15 @@ +package strcomp + +func EqualFold(a, b string) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i]|0x20 != b[i]|0x20 { + return false + } + } + + return true +} From 61c9ac0c65848f779ba4e77993d08b2f793be241 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:01:49 +0200 Subject: [PATCH 3/7] removed old bench cases, added new ones new ones are more representative, and more huge --- .../parser/http1/requestsparserbench_test.go | 64 +++++++++++++------ 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/internal/parser/http1/requestsparserbench_test.go b/internal/parser/http1/requestsparserbench_test.go index 4cd581c9..1f024b75 100644 --- a/internal/parser/http1/requestsparserbench_test.go +++ b/internal/parser/http1/requestsparserbench_test.go @@ -1,6 +1,8 @@ package http1 import ( + "strconv" + "strings" "testing" ) @@ -17,38 +19,60 @@ func BenchmarkHttpRequestsParser_Parse_GET(b *testing.B) { } }) - b.Run("BiggerGET", func(b *testing.B) { - b.SetBytes(int64(len(biggerGET))) + b.Run("5 headers", func(b *testing.B) { + request := generateRequest(5, "www.google.com", 13) + b.SetBytes(int64(len(request))) b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, _ = parser.Parse(biggerGET) + _, _, _ = parser.Parse(request) parser.Release() } }) - tenHeaders := []byte( - "GET / HTTP/1.1\r\n" + - "Header1: value1\r\n" + - "Header2: value2\r\n" + - "Header3: value3\r\n" + - "ROFL Header: ROFL value\r\n" + - "Header-5: value 5\r\n" + - "Header-6: haha lol\r\n" + - "Header-7: rolling out of laugh\r\n" + - "Header-8: sometimes I just wanna chicken fries\r\n" + - "Header-9: but having only fried potatoes instead\r\n" + - "Header-10: and this is sometimes really annoying\r\n" + - "\r\n", - ) + b.Run("10 headers", func(b *testing.B) { + request := generateRequest(10, "www.google.com", 13) + b.SetBytes(int64(len(request))) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + _, _, err := parser.Parse(request) + if err != nil { + panic(err) + } + parser.Release() + } + }) - b.Run("TenHeaders", func(b *testing.B) { - b.SetBytes(int64(len(tenHeaders))) + b.Run("50 headers", func(b *testing.B) { + request := generateRequest(50, "www.google.com", 13) + b.SetBytes(int64(len(request))) b.ResetTimer() for i := 0; i < b.N; i++ { - _, _, _ = parser.Parse(tenHeaders) + _, _, err := parser.Parse(request) + if err != nil { + panic(err) + } parser.Release() } }) } + +func generateRequest(headersNum int, hostValue string, contentLengthValue int) (request []byte) { + request = append(request, + "GET /"+strings.Repeat("a", 500)+" HTTP/1.1\r\n"..., + ) + + for i := 0; i < headersNum-2; i++ { + request = append(request, + "some-random-header-name-nobody-cares-about"+strconv.Itoa(i)+": "+ + strings.Repeat("b", 100)+"\r\n"..., + ) + } + + request = append(request, "Host: "+hostValue+"\r\n"...) + request = append(request, "Content-Length: "+strconv.Itoa(contentLengthValue)+"\r\n"...) + + return append(request, '\r', '\n') +} From cb31f95c082c0ccd1995e8b77af5f60a09f22d51 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:02:15 +0200 Subject: [PATCH 4/7] removed fragment from the path. It is simply ignored now --- http/request.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/http/request.go b/http/request.go index b7e6e196..0ef70acd 100644 --- a/http/request.go +++ b/http/request.go @@ -19,13 +19,10 @@ type ( Params = map[string]string Path struct { - String string - Params Params - Query query.Query - Fragment Fragment + String string + Params Params + Query query.Query } - - Fragment = string ) // Request struct represents http request @@ -124,7 +121,6 @@ func (r *Request) WasHijacked() bool { // Clear resets request headers and reads body into nowhere until completed. // It is implemented to clear the request object between requests func (r *Request) Clear() (err error) { - r.Path.Fragment = "" r.Path.Query.Set(nil) r.Ctx = context.Background() r.response = r.response.Clear() From 8db5954f0e65ac168da16e32baeeddd7435f3c95 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:03:42 +0200 Subject: [PATCH 5/7] added manual case-insensitive keys comparing as parser since now doesn't normalize headers anymore, we have to do this manually while getting header values --- http/headers/headers.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/http/headers/headers.go b/http/headers/headers.go index e86585ab..55dda702 100644 --- a/http/headers/headers.go +++ b/http/headers/headers.go @@ -1,5 +1,7 @@ package headers +import "github.com/indigo-web/indigo/internal/strcomp" + type Iterator[T any] func() (element T, continue_ bool) // Headers is a struct that encapsulates headers map from user, allowing only @@ -48,7 +50,7 @@ func (h *Headers) Value(key string) string { // ValueOr returns a header value func (h *Headers) ValueOr(key, or string) string { for i := 0; i < len(h.headers); i += 2 { - if h.headers[i] == key { + if strcomp.EqualFold(h.headers[i], key) { return h.headers[i+1] } } @@ -65,7 +67,7 @@ func (h *Headers) ValuesIter(key string) Iterator[string] { } for ; offset < len(h.headers); offset += 2 { - if h.headers[offset] == key { + if strcomp.EqualFold(h.headers[offset], key) { value := h.headers[offset+1] offset += 2 @@ -97,7 +99,7 @@ func (h *Headers) Add(key, value string) { // Has returns true or false depending on whether such a key exists func (h *Headers) Has(key string) bool { for i := 0; i < len(h.headers); i += 2 { - if h.headers[i] == key { + if strcomp.EqualFold(h.headers[i], key) { return true } } @@ -137,7 +139,7 @@ func collectIterator(iter Iterator[string]) (values []string) { func contains(elements []string, key string) bool { for i := range elements { - if elements[i] == key { + if strcomp.EqualFold(elements[i], key) { return true } } From c698d066bce5e02ab2a571d5c1bc78ce3d76b00e Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:04:01 +0200 Subject: [PATCH 6/7] removed fragments from the trace back-rendering --- router/inbuilt/trace.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/router/inbuilt/trace.go b/router/inbuilt/trace.go index 9f7d6bac..4dd5b00f 100644 --- a/router/inbuilt/trace.go +++ b/router/inbuilt/trace.go @@ -27,9 +27,9 @@ func renderHTTPRequest(request *http.Request, buff []byte) []byte { buff = append(buff, bytes.TrimSpace(proto.ToBytes(request.Proto))...) buff = append(buff, httpchars.CRLF...) buff = requestHeaders(request, buff) - buff = append(buff, "content-length: 0\r\n"...) + buff = append(buff, "Content-Length: 0\r\n\r\n"...) - return append(buff, httpchars.CRLF...) + return buff } func requestURI(request *http.Request, buff []byte) []byte { @@ -40,11 +40,6 @@ func requestURI(request *http.Request, buff []byte) []byte { buff = append(buff, query...) } - if len(request.Path.Fragment) > 0 { - buff = append(buff, '#') - buff = append(buff, request.Path.Fragment...) - } - return buff } From 4f161ab673bc4379779bd00b179accf989be79f2 Mon Sep 17 00:00:00 2001 From: fakefloordiv Date: Sun, 30 Jul 2023 01:07:55 +0200 Subject: [PATCH 7/7] updated pre-allocation and top sizes for header values buffer --- settings/settings.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/settings/settings.go b/settings/settings.go index 6bb883d3..d9ecc5e3 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -129,10 +129,11 @@ func Default() Settings { MaxKeyLength: 100, // 100 bytes MaxValueLength: 8192, // 8 kilobytes (just like nginx) ValueSpace: HeadersValuesSpace{ - // for simple requests without many header values this will be enough, I hope - Default: 1024, - // 128kb as a limit of amount of memory for header values storing - Maximal: 128 * 1024, + // for simple requests without many heavy-weighted headers must be enough + // to avoid a relatively big amount of re-allocations + Default: 2048, + // 64kb as a limit of amount of memory for header values storing + Maximal: 64 * 1024, }, }, URL: URL{