Merge pull request #378 from projectdiscovery/fix_body_too_large_err

fix `http: request body too large` err

http/normalization.go

@@ -48,7 +48,7 @@ func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) {
 				return errors.Wrap(err, "could not read response body after gzip error")
 			}
 		}
-		if stringsutil.ContainsAny(err.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled") {
+		if stringsutil.ContainsAnyI(err.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled", "http: request body too large") {
 			// keep partial body and continue (skip error) (add meta header in response for debugging)
 			if response.Header == nil {
 				response.Header = make(http.Header)

http/response.go

@@ -5,10 +5,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+
+	"github.com/docker/go-units"
 )
 
 var (
-	MaxBodyRead = int64(4 << 20) // 4MB
+	MaxBodyRead, _ = units.FromHumanSize("4mb")
 )
 
 // DumpResponseIntoBuffer dumps a http response without allocating a new buffer

reader/conn_read.go

@@ -7,13 +7,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/go-units"
 	contextutil "github.com/projectdiscovery/utils/context"
 	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
-const (
+var (
 	// although this is more than enough for most cases
-	MaxReadSize = 1 << 23 // 8MB
+	MaxReadSize, _ = units.FromHumanSize("8mb")
 )
 
 var (