From 5c4bc1a50afb2634fbc54a3de268bced14f8b14c Mon Sep 17 00:00:00 2001 From: nanmu42 Date: Sun, 29 Nov 2020 15:17:27 +0800 Subject: [PATCH 1/3] vendor: bump dependency version --- go.mod | 6 +++--- go.sum | 52 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 3b4039d..ccea30d 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/nanmu42/gzip go 1.13 require ( - github.com/gin-gonic/gin v1.5.0 + github.com/gin-gonic/gin v1.6.3 github.com/klauspost/compress v1.11.3 - github.com/signalsciences/ac v1.1.0 - github.com/stretchr/testify v1.4.0 + github.com/signalsciences/ac v1.2.0 + github.com/stretchr/testify v1.6.1 ) diff --git a/go.sum b/go.sum index 49d41ca..fbfd78c 100644 --- a/go.sum +++ b/go.sum @@ -3,45 +3,55 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.5.0 h1:fi+bqFAx/oLK54somfCtEZs9HeH1LHVoEPUgARpTqyc= -github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= -github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= -github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/signalsciences/ac v1.1.0 h1:3PS8uD5khoTgnnF10aCswUAVDgW8o30SGcN6XUv27yU= -github.com/signalsciences/ac v1.1.0/go.mod h1:jnlGjtNM8dyGcnOdZjY35vHmUtOn5M5K4U+BzcVPjN0= +github.com/signalsciences/ac v1.2.0 h1:6UcueKRSJn7iHhq1vKU7R0EVhzCJf77tD6HjAGcGDSs= +github.com/signalsciences/ac v1.2.0/go.mod h1:jnlGjtNM8dyGcnOdZjY35vHmUtOn5M5K4U+BzcVPjN0= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc= -gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From b0483ce43b4f98f3aa48905fe994dfd13ef5cfa6 Mon Sep 17 00:00:00 2001 From: nanmu42 Date: Sun, 29 Nov 2020 15:41:12 +0800 Subject: [PATCH 2/3] doc: acknoledgement and performance correction --- README.Chinese.md | 20 ++++++++- README.md | 20 ++++++++- docs/benchmarks.md | 108 --------------------------------------------- 3 files changed, 38 insertions(+), 110 deletions(-) delete mode 100644 docs/benchmarks.md diff --git a/README.Chinese.md b/README.Chinese.md index fd57051..09b8246 100644 --- a/README.Chinese.md +++ b/README.Chinese.md @@ -91,7 +91,23 @@ handler := gzip.NewHandler(gzip.Config{ # 效率 -本中间件经过了性能调优，以确保高效运行，[查看benchmark](https://github.com/nanmu42/gzip/blob/master/docs/benchmarks.md)。 +* 当返回体积不大时，Handler会智能地跳过压缩，这个过程带来的代价可以忽略不记； +* 当返回体积足够大时，Handler会进行gzip压缩，这个过程有着合理的代价。 + +``` +$ go test -benchmem -bench . +goos: linux +goarch: amd64 +pkg: github.com/nanmu42/gzip +BenchmarkSoleGin_SmallPayload-4 4104684 276 ns/op 64 B/op 2 allocs/op +BenchmarkGinWithDefaultHandler_SmallPayload-4 1683307 707 ns/op 96 B/op 3 allocs/op +BenchmarkSoleGin_BigPayload-4 4198786 274 ns/op 64 B/op 2 allocs/op +BenchmarkGinWithDefaultHandler_BigPayload-4 44780 27636 ns/op 190 B/op 5 allocs/op +PASS +ok github.com/nanmu42/gzip 6.373s +``` + +注：由于[一个笨拙的人为错误](https://github.com/nanmu42/gzip/pull/3#issuecomment-735352715)，`v1.0.0`以及更早版本的评测指标是错误的，不具有参考意义。 # 局限性 @@ -111,6 +127,8 @@ API已经稳定，`1.x`版本中的更新会向前兼容。 * https://blog.cloudflare.com/results-experimenting-brotli/ * https://support.cloudflare.com/hc/en-us/articles/200168396-What-will-Cloudflare-compress- +本项目使用[klauspost的compress库](https://github.com/klauspost/compress)中的gzip压缩实现。 + Logo在[Gopherize.me](https://gopherize.me/)生成。 # License diff --git a/README.md b/README.md index 086de80..c24fb96 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,23 @@ You may define one that specially suits your need. # Performance -This middleware is fine-tuned so that little overhead is added to your program. See [benchmarks](https://github.com/nanmu42/gzip/blob/master/docs/benchmarks.md). +* When response payload is small, the handler is smart enough to skip compression automatically, which takes neglectable overhead. +* At the time when the payload is big enough, gzip kicks in and there is a reasonable price. + +``` +$ go test -benchmem -bench . +goos: linux +goarch: amd64 +pkg: github.com/nanmu42/gzip +BenchmarkSoleGin_SmallPayload-4 4104684 276 ns/op 64 B/op 2 allocs/op +BenchmarkGinWithDefaultHandler_SmallPayload-4 1683307 707 ns/op 96 B/op 3 allocs/op +BenchmarkSoleGin_BigPayload-4 4198786 274 ns/op 64 B/op 2 allocs/op +BenchmarkGinWithDefaultHandler_BigPayload-4 44780 27636 ns/op 190 B/op 5 allocs/op +PASS +ok github.com/nanmu42/gzip 6.373s +``` + +Note: due to [an awkward man-mistake](https://github.com/nanmu42/gzip/pull/3#issuecomment-735352715), benchmark of and before `v1.0.0` are not accurate. # Limitation @@ -112,6 +128,8 @@ During the development of this work, the author took following works/materials a * https://blog.cloudflare.com/results-experimenting-brotli/ * https://support.cloudflare.com/hc/en-us/articles/200168396-What-will-Cloudflare-compress- +This package uses [klauspost's compress package](https://github.com/klauspost/compress) to handle gzip compression. + Logo generated at [Gopherize.me](https://gopherize.me/). # License diff --git a/docs/benchmarks.md b/docs/benchmarks.md deleted file mode 100644 index 4572f72..0000000 --- a/docs/benchmarks.md +++ /dev/null @@ -1,108 +0,0 @@ -# Command - -All benchmarks were running on the same machine. - -```bash -go test -benchmem -cpuprofile cpu.prof -memprofile mem.prof -bench=. -``` - -# v0.1.0 - -v0.1.0 gets things working. - -go version go1.13.4 linux/amd64 - -``` -goos: linux -goarch: amd64 -pkg: github.com/nanmu42/gzip -BenchmarkSoleGin_SmallPayload-12 7222698 244 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_SmallPayload-12 1000000 1076 ns/op 224 B/op 6 allocs/op -BenchmarkSoleGin_BigPayload-12 6688381 265 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_BigPayload-12 1000000 1063 ns/op 224 B/op 6 allocs/op -PASS -ok github.com/nanmu42/gzip 6.222s -``` - -# v0.2.0 - -v0.2.0 uses ahocorasick in substring matching. - -go version go1.13.4 linux/amd64 - -``` -goos: linux -goarch: amd64 -pkg: github.com/nanmu42/gzip -BenchmarkSoleGin_SmallPayload-12 6769252 182 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_SmallPayload-12 1410784 740 ns/op 224 B/op 6 allocs/op -BenchmarkSoleGin_BigPayload-12 7300908 218 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_BigPayload-12 2312258 726 ns/op 224 B/op 6 allocs/op -PASS -ok github.com/nanmu42/gzip 7.428s -``` - -# v0.3.0 - -v0.3.0 reuses writerWrapper to gain less allocations and GC pressure. - -go version go1.13.4 linux/amd64 - -``` -goos: linux -goarch: amd64 -pkg: github.com/nanmu42/gzip -BenchmarkSoleGin_SmallPayload-12 7376715 194 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_SmallPayload-12 2475199 466 ns/op 96 B/op 3 allocs/op -BenchmarkSoleGin_BigPayload-12 6572848 191 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_BigPayload-12 2991879 398 ns/op 96 B/op 3 allocs/op -PASS -ok github.com/nanmu42/gzip 6.425s -``` - -# v0.4.0 - -v0.4.0 fixes panic on Gin's no route error. - -# v0.5.0 - -v0.5.0 fixes panic on second calling to Write(). - -``` -goos: linux -goarch: amd64 -pkg: github.com/nanmu42/gzip -BenchmarkSoleGin_SmallPayload-12 7490284 201 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_SmallPayload-12 2292319 501 ns/op 96 B/op 3 allocs/op -BenchmarkSoleGin_BigPayload-12 6403441 190 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_BigPayload-12 2951451 410 ns/op 96 B/op 3 allocs/op -PASS -ok github.com/nanmu42/gzip 6.620s -``` - -# v0.6.0 - -v0.6.0 fixes wrong status code handling CORS OPTIONS request by gin's other middleware. - -# v0.7.0 - -* writerWrapper: buffer writes to decide whether use gzip or not -* writerWrapper: detect Content-Type if there's none -* ginGzipWriter: full implementation for gin.ResponseWriter excluding Pusher() - - -``` -goos: linux -goarch: amd64 -pkg: github.com/nanmu42/gzip -BenchmarkSoleGin_SmallPayload-12 7900057 184 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_SmallPayload-12 2171088 510 ns/op 96 B/op 3 allocs/op -BenchmarkSoleGin_BigPayload-12 7402651 184 ns/op 64 B/op 2 allocs/op -BenchmarkGinWithDefaultHandler_BigPayload-12 2911062 404 ns/op 96 B/op 3 allocs/op -PASS -ok github.com/nanmu42/gzip 6.634s -``` - -# v0.8.0 - -v0.8.0 considers second(and furthermore) calls to writeWrapper's WriteHeader() valid From 96ed5c7bdd89fb47cfb2f6be0c8a0e4797e65303 Mon Sep 17 00:00:00 2001 From: nanmu42 Date: Sun, 29 Nov 2020 15:44:46 +0800 Subject: [PATCH 3/3] refactor: simplify writerWrapper.enoughContentLength() --- writerwrapper.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/writerwrapper.go b/writerwrapper.go index 9559d3c..f8c938c 100644 --- a/writerwrapper.go +++ b/writerwrapper.go @@ -178,15 +178,10 @@ func (w *writerWrapper) writeBuffer(data []byte) (fit bool) { } func (w *writerWrapper) enoughContentLength() bool { - var ( - header = w.Header() - _, haveCl = header["Content-Length"] - contentLength int64 - ) - if haveCl { - contentLength, _ = strconv.ParseInt(header.Get("Content-Length"), 10, 64) + contentLength, err := strconv.ParseInt(w.Header().Get("Content-Length"), 10, 64) + if err != nil { + return false } - if contentLength != 0 && contentLength >= w.MinContentLength { return true }