add content len field (#1033)

projectdiscovery · Oct 31, 2024 · 6de8746 · 6de8746
1 parent fe27754
commit 6de8746
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 9 deletions.
diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go
@@ -105,16 +105,18 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re
 		bodyReader, _ := goquery.NewDocumentFromReader(bytes.NewReader(body))
 		technologies := c.Options.Wappalyzer.Fingerprint(headers, body)
 		resp := &navigation.Response{
-			Resp:         httpresp,
-			Body:         string(body),
-			Reader:       bodyReader,
-			Depth:        depth,
-			RootHostname: s.Hostname,
-			Technologies: mapsutil.GetKeys(technologies),
-			StatusCode:   statusCode,
-			Headers:      utils.FlattenHeaders(headers),
-			Raw:          string(rawBytesResponse),
+			Resp:          httpresp,
+			Body:          string(body),
+			Reader:        bodyReader,
+			Depth:         depth,
+			RootHostname:  s.Hostname,
+			Technologies:  mapsutil.GetKeys(technologies),
+			StatusCode:    statusCode,
+			Headers:       utils.FlattenHeaders(headers),
+			Raw:           string(rawBytesResponse),
+			ContentLength: httpresp.ContentLength,
 		}
+		response.ContentLength = resp.ContentLength
 
 		requestHeaders := make(map[string][]string)
 		for name, value := range e.Request.Headers {

diff --git a/pkg/engine/standard/crawl.go b/pkg/engine/standard/crawl.go
@@ -96,6 +96,7 @@ func (c *Crawler) makeRequest(s *common.CrawlSession, request *navigation.Reques
 	}
 
 	resp.ContentLength = int64(len(data))
+	response.ContentLength = resp.ContentLength
 
 	rawResponseBytes, _ := httputil.DumpResponse(resp, true)
 	response.Raw = string(rawResponseBytes)

diff --git a/pkg/navigation/response.go b/pkg/navigation/response.go
@@ -34,6 +34,7 @@ type Response struct {
 	StatusCode         int               `json:"status_code,omitempty"`
 	Headers            Headers           `json:"headers,omitempty"`
 	Body               string            `json:"body,omitempty"`
+	ContentLength      int64             `json:"content_length,omitempty"`
 	RootHostname       string            `json:"-"`
 	Technologies       []string          `json:"technologies,omitempty"`
 	Raw                string            `json:"raw,omitempty"`