Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add gzip HTTP handler/client wrapper #383

Merged
merged 14 commits into from
Jun 2, 2021
Prev Previous commit
Next Next commit
Strip Accept-Ranges on compressed content
Fixes nytimes/gziphandler#83

Adds `KeepAcceptRanges()` if for whatever reason you would want to keep them.
  • Loading branch information
klauspost committed Jun 2, 2021
commit 7c3644ac6f49a1f7c07ff945355ade53dd924e7b
38 changes: 30 additions & 8 deletions gzhttp/gzip.go
Original file line number Diff line number Diff line change
@@ -20,6 +20,8 @@ const (
vary = "Vary"
acceptEncoding = "Accept-Encoding"
contentEncoding = "Content-Encoding"
contentRange = "Content-Range"
acceptRanges = "Accept-Ranges"
contentType = "Content-Type"
contentLength = "Content-Length"
)
@@ -51,9 +53,10 @@ type GzipResponseWriter struct {

code int // Saves the WriteHeader value.

minSize int // Specifies the minimum response size to gzip. If the response length is bigger than this value, it is compressed.
buf []byte // Holds the first part of the write before reaching the minSize or the end of the write.
ignore bool // If true, then we immediately passthru writes to the underlying ResponseWriter.
minSize int // Specifies the minimum response size to gzip. If the response length is bigger than this value, it is compressed.
buf []byte // Holds the first part of the write before reaching the minSize or the end of the write.
ignore bool // If true, then we immediately passthru writes to the underlying ResponseWriter.
keepAcceptRanges bool // Keep "Accept-Ranges" header.

contentTypeFilter func(ct string) bool // Only compress if the response is one of these content-types. All are accepted if empty.
}
@@ -86,13 +89,16 @@ func (w *GzipResponseWriter) Write(b []byte) (int, error) {
cl, _ = strconv.Atoi(w.Header().Get(contentLength))
ct = w.Header().Get(contentType)
ce = w.Header().Get(contentEncoding)
cr = w.Header().Get(contentRange)
)

// Only continue if they didn't already choose an encoding or a known unhandled content length or type.
if ce == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || w.contentTypeFilter(ct)) {
if ce == "" && cr == "" && (cl == 0 || cl >= w.minSize) && (ct == "" || w.contentTypeFilter(ct)) {
// If the current buffer is less than minSize and a Content-Length isn't set, then wait until we have more data.
if len(w.buf) < w.minSize && cl == 0 {
return len(b), nil
}

// If the Content-Length is larger than minSize or the current buffer is larger than minSize, then continue.
if cl >= w.minSize || len(w.buf) >= w.minSize {
// If a Content-Type wasn't specified, infer it from the current buffer.
@@ -132,6 +138,11 @@ func (w *GzipResponseWriter) startGzip() error {
// See: https://github.com/golang/go/issues/14975.
w.Header().Del(contentLength)

// Delete Accept-Ranges.
if !w.keepAcceptRanges {
w.Header().Del(acceptRanges)
}

// Write the header to gzip response.
if w.code != 0 {
w.ResponseWriter.WriteHeader(w.code)
@@ -297,6 +308,7 @@ func NewWrapper(opts ...option) (func(http.Handler) http.Handler, error) {
level: c.level,
minSize: c.minSize,
contentTypeFilter: c.contentTypes,
keepAcceptRanges: c.keepAcceptRanges,
}
defer gw.Close()

@@ -345,10 +357,11 @@ func (pct parsedContentType) equals(mediaType string, params map[string]string)

// Used for functional configuration.
type config struct {
minSize int
level int
writer writer.GzipWriterFactory
contentTypes func(ct string) bool
minSize int
level int
writer writer.GzipWriterFactory
contentTypes func(ct string) bool
keepAcceptRanges bool
}

func (c *config) validate() error {
@@ -457,6 +470,15 @@ func ExceptContentTypes(types []string) option {
}
}

// KeepAcceptRanges will keep Accept-Ranges header on gzipped responses.
// This will likely break ranged requests since that cannot be transparently
// handled by the filter.
func KeepAcceptRanges() option {
return func(c *config) {
c.keepAcceptRanges = true
}
}

// ContentTypeFilter allows adding a custom content type filter.
//
// The supplied function must return true/false to indicate if content
66 changes: 66 additions & 0 deletions gzhttp/gzip_test.go
Original file line number Diff line number Diff line change
@@ -114,6 +114,72 @@ func TestGzipHandlerAlreadyCompressed(t *testing.T) {
assertEqual(t, testBody, res.Body.String())
}

func TestGzipHandlerRangeReply(t *testing.T) {
handler := GzipHandler(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Range", "bytes 0-300/804")
w.WriteHeader(http.StatusOK)
w.Write([]byte(testBody))
}))
req, _ := http.NewRequest("GET", "/gzipped", nil)
req.Header.Set("Accept-Encoding", "gzip")

resp := httptest.NewRecorder()
handler.ServeHTTP(resp, req)
res := resp.Result()
assertEqual(t, 200, res.StatusCode)
assertEqual(t, "", res.Header.Get("Content-Encoding"))
assertEqual(t, testBody, resp.Body.String())
}

func TestGzipHandlerAcceptRange(t *testing.T) {
handler := GzipHandler(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Accept-Ranges", "bytes")
w.WriteHeader(http.StatusOK)
w.Write([]byte(testBody))
}))
req, _ := http.NewRequest("GET", "/gzipped", nil)
req.Header.Set("Accept-Encoding", "gzip")

resp := httptest.NewRecorder()
handler.ServeHTTP(resp, req)
res := resp.Result()
assertEqual(t, 200, res.StatusCode)
assertEqual(t, "gzip", res.Header.Get("Content-Encoding"))
assertEqual(t, "", res.Header.Get("Accept-Ranges"))
zr, err := gzip.NewReader(resp.Body)
assertNil(t, err)
got, err := ioutil.ReadAll(zr)
assertNil(t, err)
assertEqual(t, testBody, string(got))
}

func TestGzipHandlerKeepAcceptRange(t *testing.T) {
wrapper, err := NewWrapper(KeepAcceptRanges())
assertNil(t, err)
handler := wrapper(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Accept-Ranges", "bytes")
w.WriteHeader(http.StatusOK)
w.Write([]byte(testBody))
}))
req, _ := http.NewRequest("GET", "/gzipped", nil)
req.Header.Set("Accept-Encoding", "gzip")

resp := httptest.NewRecorder()
handler.ServeHTTP(resp, req)
res := resp.Result()
assertEqual(t, 200, res.StatusCode)
assertEqual(t, "gzip", res.Header.Get("Content-Encoding"))
assertEqual(t, "bytes", res.Header.Get("Accept-Ranges"))
zr, err := gzip.NewReader(resp.Body)
assertNil(t, err)
got, err := ioutil.ReadAll(zr)
assertNil(t, err)
assertEqual(t, testBody, string(got))
}

func TestNewGzipLevelHandler(t *testing.T) {
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)