From 97bba324dca606bc3b8ae160de637edbca580d02 Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Tue, 31 Dec 2024 21:41:43 +1000 Subject: [PATCH 1/7] ext/har: add HAR logger extension Port HAR logging from abourget/goproxy to ext package. Closes #609 --- ext/go.mod | 4 +- ext/go.sum | 2 - ext/har/logger.go | 110 +++++++++++++ ext/har/logger_test.go | 114 +++++++++++++ ext/har/types.go | 365 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 592 insertions(+), 3 deletions(-) create mode 100644 ext/har/logger.go create mode 100644 ext/har/logger_test.go create mode 100644 ext/har/types.go diff --git a/ext/go.mod b/ext/go.mod index 9c7d3945..1e3a2ffc 100644 --- a/ext/go.mod +++ b/ext/go.mod @@ -2,8 +2,10 @@ module github.com/elazarl/goproxy/ext go 1.20 +replace github.com/elazarl/goproxy => ../ + require ( - github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c + github.com/elazarl/goproxy v0.0.0 golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 ) diff --git a/ext/go.sum b/ext/go.sum index b9cadd46..845330e7 100644 --- a/ext/go.sum +++ b/ext/go.sum @@ -1,5 +1,3 @@ -github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c h1:yWAGp1CjD1mQGLUsADqPn5s1n2AkGAX33XLDUgoXzyo= -github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c/go.mod h1:P73liMk9TZCyF9fXG/RyMeSizmATvpvy3ZS61/1eXn4= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= diff --git a/ext/har/logger.go b/ext/har/logger.go new file mode 100644 index 00000000..eefafdfb --- /dev/null +++ b/ext/har/logger.go @@ -0,0 +1,110 @@ +package har + + +import ( + "encoding/json" + "net/http" + "os" + "sync" + "time" + + "github.com/elazarl/goproxy" +) + +// Logger implements a HAR logging extension for goproxy +type Logger struct { + mu sync.Mutex + har *Har + captureContent bool +} + +// NewLogger creates a new HAR logger instance +func NewLogger() *Logger { + return &Logger{ + har: New(), + } +} + +// OnRequest handles incoming HTTP requests +func (l *Logger) OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + // Store the start time in context for later use + if ctx != nil { + ctx.UserData = time.Now() + } + return req, nil +} + +// OnResponse handles HTTP responses +func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + if resp == nil || ctx == nil || ctx.Req == nil || ctx.UserData == nil { + return resp + } + + startTime, ok := ctx.UserData.(time.Time) + if !ok { + return resp + } + + // Create HAR entry + entry := Entry{ + StartedDateTime: startTime, + Time: time.Since(startTime).Milliseconds(), + Request: ParseRequest(ctx.Req, l.captureContent), + Response: ParseResponse(resp, l.captureContent), + Cache: Cache{}, + Timings: Timings{ + Send: 0, + Wait: time.Since(startTime).Milliseconds(), + Receive: 0, + }, + } + + // Add server IP + entry.FillIPAddress(ctx.Req) + + // Add to HAR log thread-safely + l.mu.Lock() + l.har.AppendEntry(entry) + l.mu.Unlock() + + return resp +} + +// SetCaptureContent enables or disables request/response body capture +func (l *Logger) SetCaptureContent(capture bool) { + l.mu.Lock() + defer l.mu.Unlock() + l.captureContent = capture +} + +// SaveToFile writes the current HAR log to a file +func (l *Logger) SaveToFile(filename string) error { + l.mu.Lock() + defer l.mu.Unlock() + + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + return encoder.Encode(l.har) +} + +// Clear resets the HAR log +func (l *Logger) Clear() { + l.mu.Lock() + defer l.mu.Unlock() + l.har = New() +} + +// GetEntries returns a copy of the current HAR entries +func (l *Logger) GetEntries() []Entry { + l.mu.Lock() + defer l.mu.Unlock() + entries := make([]Entry, len(l.har.Log.Entries)) + copy(entries, l.har.Log.Entries) + return entries +} diff --git a/ext/har/logger_test.go b/ext/har/logger_test.go new file mode 100644 index 00000000..5fe0286a --- /dev/null +++ b/ext/har/logger_test.go @@ -0,0 +1,114 @@ + +package har_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/har" +) + +type ConstantHandler string + +func (h ConstantHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, string(h)) +} + +func oneShotProxy(proxy *goproxy.ProxyHttpServer) (client *http.Client, s *httptest.Server) { + s = httptest.NewServer(proxy) + + proxyUrl, _ := url.Parse(s.URL) + tr := &http.Transport{Proxy: http.ProxyURL(proxyUrl)} + client = &http.Client{Transport: tr} + return +} + +func TestHarLogger(t *testing.T) { + // Create a response we expect + expected := "hello world" + background := httptest.NewServer(ConstantHandler(expected)) + defer background.Close() + + // Set up the proxy with HAR logger + proxy := goproxy.NewProxyHttpServer() + logger := har.NewLogger() + logger.SetCaptureContent(true) + + proxy.OnRequest().DoFunc(logger.OnRequest) + proxy.OnResponse().DoFunc(logger.OnResponse) + + client, proxyserver := oneShotProxy(proxy) + defer proxyserver.Close() + + // Make a request + resp, err := client.Get(background.URL) + if err != nil { + t.Fatal(err) + } + + // Read the response + msg, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + if string(msg) != expected { + t.Errorf("Expected '%s', actual '%s'", expected, string(msg)) + } + + // Test POST request with content + postData := "test=value" + req, err := http.NewRequest("POST", background.URL, bytes.NewBufferString(postData)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err = client.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + + // Save HAR file and verify content + tmpfile := "test.har" + err = logger.SaveToFile(tmpfile) + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile) + + // Read and verify HAR content + harData, err := os.ReadFile(tmpfile) + if err != nil { + t.Fatal(err) + } + + var harLog har.Har + if err := json.Unmarshal(harData, &harLog); err != nil { + t.Fatal(err) + } + + // Verify we captured both requests + if len(harLog.Log.Entries) != 2 { + t.Errorf("Expected 2 entries in HAR log, got %d", len(harLog.Log.Entries)) + } + + // Verify GET request + if harLog.Log.Entries[0].Request.Method != "GET" { + t.Errorf("Expected GET request, got %s", harLog.Log.Entries[0].Request.Method) + } + + // Verify POST request + if harLog.Log.Entries[1].Request.Method != "POST" { + t.Errorf("Expected POST request, got %s", harLog.Log.Entries[1].Request.Method) + } +} diff --git a/ext/har/types.go b/ext/har/types.go new file mode 100644 index 00000000..ff94584c --- /dev/null +++ b/ext/har/types.go @@ -0,0 +1,365 @@ +// Original implementation from abourget/goproxy, adapted for use as an extension. +// HAR specification: http://www.softwareishard.com/blog/har-12-spec/ +package har + +import ( + "bytes" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +var startingEntrySize int = 1000 + +type Har struct { + Log Log `json:"log"` +} + +type Log struct { + Version string `json:"version"` + Creator Creator `json:"creator"` + Browser *Browser `json:"browser,omitempty"` + Pages []Page `json:"pages,omitempty"` + Entries []Entry `json:"entries"` + Comment string `json:"comment,omitempty"` +} + +func New() *Har { + har := &Har{ + Log: Log{ + Version: "1.2", + Creator: Creator{ + Name: "GoProxy", + Version: "12345", + }, + Pages: make([]Page, 0, 10), + Entries: makeNewEntries(), + }, + } + return har +} + +func (har *Har) AppendEntry(entry ...Entry) { + har.Log.Entries = append(har.Log.Entries, entry...) +} + +func (har *Har) AppendPage(page ...Page) { + har.Log.Pages = append(har.Log.Pages, page...) +} + +func makeNewEntries() []Entry { + return make([]Entry, 0, startingEntrySize) +} + +type Creator struct { + Name string `json:"name"` + Version string `json:"version"` + Comment string `json:"comment,omitempty"` +} + +type Browser struct { + Name string `json:"name"` + Version string `json:"version"` + Comment string `json:"comment,omitempty"` +} + +type Page struct { + ID string `json:"id,omitempty"` + StartedDateTime time.Time `json:"startedDateTime"` + Title string `json:"title"` + PageTimings PageTimings `json:"pageTimings"` + Comment string `json:"comment,omitempty"` +} + +type Entry struct { + PageRef string `json:"pageref,omitempty"` + StartedDateTime time.Time `json:"startedDateTime"` + Time int64 `json:"time"` + Request *Request `json:"request"` + Response *Response `json:"response"` + Cache Cache `json:"cache"` + Timings Timings `json:"timings"` + ServerIpAddress string `json:"serverIpAddress,omitempty"` + Connection string `json:"connection,omitempty"` + Comment string `json:"comment,omitempty"` +} + +type Cache struct { + BeforeRequest *CacheEntry `json:"beforeRequest,omitempty"` + AfterRequest *CacheEntry `json:"afterRequest,omitempty"` +} + +type CacheEntry struct { + Expires string `json:"expires,omitempty"` + LastAccess string `json:"lastAccess"` + ETag string `json:"eTag"` + HitCount int `json:"hitCount"` + Comment string `json:"comment,omitempty"` +} + +type Request struct { + Method string `json:"method"` + Url string `json:"url"` + HttpVersion string `json:"httpVersion"` + Cookies []Cookie `json:"cookies"` + Headers []NameValuePair `json:"headers"` + QueryString []NameValuePair `json:"queryString"` + PostData *PostData `json:"postData,omitempty"` + BodySize int64 `json:"bodySize"` + HeadersSize int64 `json:"headersSize"` +} + +func ParseRequest(req *http.Request, captureContent bool) *Request { + if req == nil { + return nil + } + harRequest := Request{ + Method: req.Method, + Url: req.URL.String(), + HttpVersion: req.Proto, + Cookies: parseCookies(req.Cookies()), + Headers: parseStringArrMap(req.Header), + QueryString: parseStringArrMap((req.URL.Query())), + BodySize: req.ContentLength, + HeadersSize: calcHeaderSize(req.Header), + } + + if captureContent && (req.Method == "POST" || req.Method == "PUT") { + harRequest.PostData = parsePostData(req) + } + + return &harRequest +} + +func (harEntry *Entry) FillIPAddress(req *http.Request) { + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + host = req.URL.Host + } + if ip := net.ParseIP(host); ip != nil { + harEntry.ServerIpAddress = string(ip) + } + + if ipaddr, err := net.LookupIP(host); err == nil { + for _, ip := range ipaddr { + if ip.To4() != nil { + harEntry.ServerIpAddress = ip.String() + return + } + } + } +} + +func calcHeaderSize(header http.Header) int64 { + headerSize := 0 + for headerName, headerValues := range header { + headerSize += len(headerName) + 2 + for _, v := range headerValues { + headerSize += len(v) + } + } + return int64(headerSize) +} + +func parsePostData(req *http.Request) *PostData { + defer func() { + if e := recover(); e != nil { + log.Printf("Error parsing request to %v: %v\n", req.URL, e) + } + }() + + harPostData := new(PostData) + contentType := req.Header["Content-Type"] + if contentType == nil { + panic("Missing content type in request") + } + harPostData.MimeType = contentType[0] + + if len(req.PostForm) > 0 { + for k, vals := range req.PostForm { + for _, v := range vals { + param := PostDataParam{ + Name: k, + Value: v, + } + harPostData.Params = append(harPostData.Params, param) + } + } + } else { + str, _ := ioutil.ReadAll(req.Body) + harPostData.Text = string(str) + } + return harPostData +} + +func parseStringArrMap(stringArrMap map[string][]string) []NameValuePair { + index := 0 + harQueryString := make([]NameValuePair, len(stringArrMap)) + for k, v := range stringArrMap { + escapedKey, _ := url.QueryUnescape(k) + escapedValues, _ := url.QueryUnescape(strings.Join(v, ",")) + harNameValuePair := NameValuePair{ + Name: escapedKey, + Value: escapedValues, + } + harQueryString[index] = harNameValuePair + index++ + } + return harQueryString +} + +func parseCookies(cookies []*http.Cookie) []Cookie { + harCookies := make([]Cookie, len(cookies)) + for i, cookie := range cookies { + harCookie := Cookie{ + Name: cookie.Name, + Domain: cookie.Domain, + HttpOnly: cookie.HttpOnly, + Path: cookie.Path, + Secure: cookie.Secure, + Value: cookie.Value, + } + if !cookie.Expires.IsZero() { + harCookie.Expires = &cookie.Expires + } + harCookies[i] = harCookie + } + return harCookies +} + +type Response struct { + Status int `json:"status"` + StatusText string `json:"statusText"` + HttpVersion string `json:"httpVersion"` + Cookies []Cookie `json:"cookies"` + Headers []NameValuePair `json:"headers"` + Content Content `json:"content"` + RedirectUrl string `json:"redirectURL"` + BodySize int64 `json:"bodySize"` + HeadersSize int64 `json:"headersSize"` + Comment string `json:"comment,omitempty"` +} + +func ParseResponse(resp *http.Response, captureContent bool) *Response { + if resp == nil { + return nil + } + + statusText := resp.Status + if len(resp.Status) > 4 { + statusText = resp.Status[4:] + } + redirectURL := resp.Header.Get("Location") + harResponse := Response{ + Status: resp.StatusCode, + StatusText: statusText, + HttpVersion: resp.Proto, + Cookies: parseCookies(resp.Cookies()), + Headers: parseStringArrMap(resp.Header), + RedirectUrl: redirectURL, + BodySize: resp.ContentLength, + HeadersSize: calcHeaderSize(resp.Header), + } + + if captureContent && resp.Body != nil { + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Printf("Error reading response body: %v", err) + return &harResponse + } + // Create a new reader for the response body + resp.Body = io.NopCloser(bytes.NewBuffer(body)) + + harResponse.Content = Content{ + Size: len(body), + Text: string(body), + MimeType: resp.Header.Get("Content-Type"), + } + } + + return &harResponse +} + +func parseContent(resp *http.Response, harContent *Content) { + defer func() { + if e := recover(); e != nil { + log.Printf("Error parsing response to %v: %v\n", resp.Request.URL, e) + } + }() + + contentType := resp.Header["Content-Type"] + if contentType == nil { + panic("Missing content type in response") + } + harContent.MimeType = contentType[0] + if resp.ContentLength == 0 { + log.Println("Empty content") + return + } + + body, _ := ioutil.ReadAll(resp.Body) + harContent.Text = string(body) + harContent.Size = len(body) + return +} + +type Cookie struct { + Name string `json:"name"` + Value string `json:"value"` + Path string `json:"path,omitempty"` + Domain string `json:"domain,omitempty"` + Expires *time.Time `json:"expires,omitempty"` + HttpOnly bool `json:"httpOnly,omitempty"` + Secure bool `json:"secure,omitempty"` +} + +type NameValuePair struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type PostData struct { + MimeType string `json:"mimeType"` + Params []PostDataParam `json:"params,omitempty"` + Text string `json:"text,omitempty"` + Comment string `json:"comment,omitempty"` +} + +type PostDataParam struct { + Name string `json:"name"` + Value string `json:"value,omitempty"` + FileName string `json:"fileName,omitempty"` + ContentType string `json:"contentType,omitempty"` + Comment string `json:"comment,omitempty"` +} + +type Content struct { + Size int `json:"size"` + Compression int `json:"compression,omitempty"` + MimeType string `json:"mimeType"` + Text string `json:"text,omitempty"` + Encoding string `json:"encoding,omitempty"` + Comment string `json:"comment,omitempty"` +} + +type PageTimings struct { + OnContentLoad int64 `json:"onContentLoad"` + OnLoad int64 `json:"onLoad"` + Comment string `json:"comment,omitempty"` +} + +type Timings struct { + Dns int64 `json:"dns,omitempty"` + Blocked int64 `json:"blocked,omitempty"` + Connect int64 `json:"connect,omitempty"` + Send int64 `json:"send"` + Wait int64 `json:"wait"` + Receive int64 `json:"receive"` + Ssl int64 `json:"ssl,omitempty"` + Comment string `json:"comment,omitempty"` +} From b30490fcaf9b1541996557353b400e9ab3296a5d Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Tue, 31 Dec 2024 22:12:34 +1000 Subject: [PATCH 2/7] reverted to go.mod and go.sum files --- ext/go.mod | 4 +--- ext/go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/go.mod b/ext/go.mod index 1e3a2ffc..9c7d3945 100644 --- a/ext/go.mod +++ b/ext/go.mod @@ -2,10 +2,8 @@ module github.com/elazarl/goproxy/ext go 1.20 -replace github.com/elazarl/goproxy => ../ - require ( - github.com/elazarl/goproxy v0.0.0 + github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 ) diff --git a/ext/go.sum b/ext/go.sum index 845330e7..b9cadd46 100644 --- a/ext/go.sum +++ b/ext/go.sum @@ -1,3 +1,5 @@ +github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c h1:yWAGp1CjD1mQGLUsADqPn5s1n2AkGAX33XLDUgoXzyo= +github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c/go.mod h1:P73liMk9TZCyF9fXG/RyMeSizmATvpvy3ZS61/1eXn4= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= From 26a6a755086f81ac16c4a5b269e2664cafe51e62 Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Wed, 1 Jan 2025 01:07:02 +1000 Subject: [PATCH 3/7] fixed code issues from pull request 610 other then testing and SaveToFile --- ext/har/logger.go | 26 ++++---- ext/har/types.go | 150 +++++++++++++++++++++------------------------- 2 files changed, 78 insertions(+), 98 deletions(-) diff --git a/ext/har/logger.go b/ext/har/logger.go index eefafdfb..b88023b9 100644 --- a/ext/har/logger.go +++ b/ext/har/logger.go @@ -22,6 +22,7 @@ type Logger struct { func NewLogger() *Logger { return &Logger{ har: New(), + captureContent: true, } } @@ -32,7 +33,7 @@ func (l *Logger) OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Requ ctx.UserData = time.Now() } return req, nil -} +} // OnResponse handles HTTP responses func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { @@ -51,7 +52,6 @@ func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Re Time: time.Since(startTime).Milliseconds(), Request: ParseRequest(ctx.Req, l.captureContent), Response: ParseResponse(resp, l.captureContent), - Cache: Cache{}, Timings: Timings{ Send: 0, Wait: time.Since(startTime).Milliseconds(), @@ -60,7 +60,7 @@ func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Re } // Add server IP - entry.FillIPAddress(ctx.Req) + entry.fillIPAddress(ctx.Req) // Add to HAR log thread-safely l.mu.Lock() @@ -70,27 +70,23 @@ func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Re return resp } -// SetCaptureContent enables or disables request/response body capture -func (l *Logger) SetCaptureContent(capture bool) { - l.mu.Lock() - defer l.mu.Unlock() - l.captureContent = capture -} - // SaveToFile writes the current HAR log to a file func (l *Logger) SaveToFile(filename string) error { l.mu.Lock() defer l.mu.Unlock() - file, err := os.Create(filename) if err != nil { return err } defer file.Close() - - encoder := json.NewEncoder(file) - encoder.SetIndent("", " ") - return encoder.Encode(l.har) + + jsonData, err := json.Marshal(l.har) + if err != nil { + return err + } + + _, err = file.Write(jsonData) + return err } // Clear resets the HAR log diff --git a/ext/har/types.go b/ext/har/types.go index ff94584c..ed4fe82d 100644 --- a/ext/har/types.go +++ b/ext/har/types.go @@ -5,7 +5,6 @@ package har import ( "bytes" "io" - "io/ioutil" "log" "net" "net/http" @@ -14,7 +13,6 @@ import ( "time" ) -var startingEntrySize int = 1000 type Har struct { Log Log `json:"log"` @@ -35,7 +33,7 @@ func New() *Har { Version: "1.2", Creator: Creator{ Name: "GoProxy", - Version: "12345", + Version: "1.0", }, Pages: make([]Page, 0, 10), Entries: makeNewEntries(), @@ -53,6 +51,7 @@ func (har *Har) AppendPage(page ...Page) { } func makeNewEntries() []Entry { + const startingEntrySize int = 1000; return make([]Entry, 0, startingEntrySize) } @@ -136,80 +135,86 @@ func ParseRequest(req *http.Request, captureContent bool) *Request { return &harRequest } -func (harEntry *Entry) FillIPAddress(req *http.Request) { - host, _, err := net.SplitHostPort(req.URL.Host) - if err != nil { - host = req.URL.Host - } - if ip := net.ParseIP(host); ip != nil { - harEntry.ServerIpAddress = string(ip) - } - - if ipaddr, err := net.LookupIP(host); err == nil { - for _, ip := range ipaddr { - if ip.To4() != nil { - harEntry.ServerIpAddress = ip.String() - return - } - } - } +func (harEntry *Entry) fillIPAddress(req *http.Request) { + host := req.URL.Hostname() + + if ip := net.ParseIP(host); ip != nil { + harEntry.ServerIpAddress = ip.String() + } } func calcHeaderSize(header http.Header) int64 { - headerSize := 0 - for headerName, headerValues := range header { - headerSize += len(headerName) + 2 - for _, v := range headerValues { - headerSize += len(v) - } - } - return int64(headerSize) + // Directly return -1 as per HAR specification + return -1 } func parsePostData(req *http.Request) *PostData { - defer func() { - if e := recover(); e != nil { - log.Printf("Error parsing request to %v: %v\n", req.URL, e) - } - }() - - harPostData := new(PostData) - contentType := req.Header["Content-Type"] - if contentType == nil { - panic("Missing content type in request") - } - harPostData.MimeType = contentType[0] - - if len(req.PostForm) > 0 { - for k, vals := range req.PostForm { - for _, v := range vals { - param := PostDataParam{ - Name: k, - Value: v, - } - harPostData.Params = append(harPostData.Params, param) - } - } - } else { - str, _ := ioutil.ReadAll(req.Body) - harPostData.Text = string(str) - } - return harPostData + harPostData := new(PostData) + + contentType := req.Header.Get("Content-Type") + if contentType == "" { + return nil + } + + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil { + log.Printf("Error parsing media type: %v", err) + return nil + } + + harPostData.MimeType = mediaType + + if err := req.ParseForm(); err != nil { + log.Printf("Error parsing form: %v", err) + return nil + } + + if len(req.PostForm) > 0 { + for k, vals := range req.PostForm { + for _, v := range vals { + param := PostDataParam{ + Name: k, + Value: v, + } + harPostData.Params = append(harPostData.Params, param) + } + } + } else { + str, err := io.ReadAll(req.Body) + if err != nil { + log.Printf("Error reading request body: %v", err) + return nil + } + harPostData.Text = string(str) + } + + return harPostData } func parseStringArrMap(stringArrMap map[string][]string) []NameValuePair { - index := 0 - harQueryString := make([]NameValuePair, len(stringArrMap)) + harQueryString := make([]NameValuePair, 0, len(stringArrMap)) + for k, v := range stringArrMap { - escapedKey, _ := url.QueryUnescape(k) - escapedValues, _ := url.QueryUnescape(strings.Join(v, ",")) + escapedKey, err := url.QueryUnescape(k) + if err != nil { + // Use original key if unescaping fails + escapedKey = k + } + + escapedValues, err := url.QueryUnescape(strings.Join(v, ",")) + if err != nil { + // Use original joined values if unescaping fails + escapedValues = strings.Join(v, ",") + } + harNameValuePair := NameValuePair{ Name: escapedKey, Value: escapedValues, } - harQueryString[index] = harNameValuePair - index++ + + harQueryString = append(harQueryString, harNameValuePair) } + return harQueryString } @@ -285,28 +290,7 @@ func ParseResponse(resp *http.Response, captureContent bool) *Response { return &harResponse } -func parseContent(resp *http.Response, harContent *Content) { - defer func() { - if e := recover(); e != nil { - log.Printf("Error parsing response to %v: %v\n", resp.Request.URL, e) - } - }() - - contentType := resp.Header["Content-Type"] - if contentType == nil { - panic("Missing content type in response") - } - harContent.MimeType = contentType[0] - if resp.ContentLength == 0 { - log.Println("Empty content") - return - } - body, _ := ioutil.ReadAll(resp.Body) - harContent.Text = string(body) - harContent.Size = len(body) - return -} type Cookie struct { Name string `json:"name"` From c0ae8a6ba9c64cf1a7ffdcbe4f491a3cf15776f1 Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Wed, 1 Jan 2025 17:03:03 +1000 Subject: [PATCH 4/7] added tesify testing and added fruther logs to ParseRequest --- ext/har/logger_test.go | 258 +++++++++++++++++++++++++++++------------ ext/har/types.go | 99 ++++++++++------ 2 files changed, 252 insertions(+), 105 deletions(-) diff --git a/ext/har/logger_test.go b/ext/har/logger_test.go index 5fe0286a..1f3fd919 100644 --- a/ext/har/logger_test.go +++ b/ext/har/logger_test.go @@ -1,114 +1,228 @@ - -package har_test +package har import ( - "bytes" "encoding/json" "io" "net/http" "net/http/httptest" "net/url" "os" + "path/filepath" + "strings" "testing" + "time" "github.com/elazarl/goproxy" - "github.com/elazarl/goproxy/ext/har" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// ConstantHandler is a simple HTTP handler that returns a constant response type ConstantHandler string func (h ConstantHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") io.WriteString(w, string(h)) } -func oneShotProxy(proxy *goproxy.ProxyHttpServer) (client *http.Client, s *httptest.Server) { - s = httptest.NewServer(proxy) +// createTestProxy sets up a test proxy with a HAR logger +func createTestProxy(logger *Logger) *httptest.Server { + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().DoFunc(logger.OnRequest) + proxy.OnResponse().DoFunc(logger.OnResponse) + return httptest.NewServer(proxy) +} + +// createProxyClient creates an HTTP client that uses the given proxy +func createProxyClient(proxyURL string) *http.Client { + proxyURLParsed, _ := url.Parse(proxyURL) + tr := &http.Transport{ + Proxy: http.ProxyURL(proxyURLParsed), + } + return &http.Client{Transport: tr} +} + - proxyUrl, _ := url.Parse(s.URL) - tr := &http.Transport{Proxy: http.ProxyURL(proxyUrl)} - client = &http.Client{Transport: tr} - return +func TestHarLoggerBasicFunctionality(t *testing.T) { + testCases := []struct { + name string + method string + body string + contentType string + expectedMethod string + }{ + { + name: "GET Request", + method: http.MethodGet, + expectedMethod: http.MethodGet, + }, + { + name: "POST Request", + method: http.MethodPost, + body: `{"test":"data"}`, + contentType: "application/json", + expectedMethod: http.MethodPost, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + background := httptest.NewServer(ConstantHandler("hello world")) + defer background.Close() + + logger := NewLogger() + proxyServer := createTestProxy(logger) + defer proxyServer.Close() + + client := createProxyClient(proxyServer.URL) + + // Prepare request + req, err := http.NewRequest(tc.method, background.URL, strings.NewReader(tc.body)) + require.NoError(t, err, "Should create request") + if tc.contentType != "" { + req.Header.Set("Content-Type", tc.contentType) + } + + // Send request and capture response + resp, err := client.Do(req) + require.NoError(t, err, "Should send request successfully") + defer resp.Body.Close() + + // Read response body + bodyBytes, _ := io.ReadAll(resp.Body) + body := string(bodyBytes) + assert.Equal(t, "hello world", body, "Response body should match") + + time.Sleep(200 * time.Millisecond) + + // Verify HAR entry + entries := logger.GetEntries() + require.Len(t, entries, 1, "Should have one log entry") + entry := entries[0] + assert.Equal(t, tc.expectedMethod, entry.Request.Method, "Request method should match") + }) + } } -func TestHarLogger(t *testing.T) { - // Create a response we expect - expected := "hello world" - background := httptest.NewServer(ConstantHandler(expected)) +func TestHarLoggerHeaders(t *testing.T) { + background := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Test-Header", "test-value") + w.Write([]byte("test")) + })) defer background.Close() - // Set up the proxy with HAR logger - proxy := goproxy.NewProxyHttpServer() - logger := har.NewLogger() - logger.SetCaptureContent(true) + logger := NewLogger() - proxy.OnRequest().DoFunc(logger.OnRequest) - proxy.OnResponse().DoFunc(logger.OnResponse) + proxyServer := createTestProxy(logger) + defer proxyServer.Close() - client, proxyserver := oneShotProxy(proxy) - defer proxyserver.Close() + client := createProxyClient(proxyServer.URL) - // Make a request - resp, err := client.Get(background.URL) - if err != nil { - t.Fatal(err) - } + req, err := http.NewRequest("GET", background.URL, nil) + require.NoError(t, err, "Should create request") + req.Header.Set("X-Custom-Header", "custom-value") - // Read the response - msg, err := io.ReadAll(resp.Body) - if err != nil { - t.Fatal(err) - } - resp.Body.Close() + resp, err := client.Do(req) + require.NoError(t, err, "Should send request") + defer resp.Body.Close() - if string(msg) != expected { - t.Errorf("Expected '%s', actual '%s'", expected, string(msg)) - } + time.Sleep(200 * time.Millisecond) + + entries := logger.GetEntries() + require.Len(t, entries, 1, "Should have one log entry") + entry := entries[0] - // Test POST request with content - postData := "test=value" - req, err := http.NewRequest("POST", background.URL, bytes.NewBufferString(postData)) - if err != nil { - t.Fatal(err) + // Convert headers to maps for easier checking + reqHeaders := make(map[string]string) + for _, h := range entry.Request.Headers { + reqHeaders[h.Name] = h.Value } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - resp, err = client.Do(req) - if err != nil { - t.Fatal(err) + assert.Equal(t, "custom-value", reqHeaders["X-Custom-Header"], "Request header value should match") + + respHeaders := make(map[string]string) + for _, h := range entry.Response.Headers { + respHeaders[h.Name] = h.Value } + assert.Equal(t, "test-value", respHeaders["X-Test-Header"], "Response header value should match") +} + +func TestHarLoggerSaveAndClear(t *testing.T) { + logger := NewLogger() + + background := httptest.NewServer(ConstantHandler("test")) + defer background.Close() + + proxyServer := createTestProxy(logger) + defer proxyServer.Close() + + client := createProxyClient(proxyServer.URL) + + resp, err := client.Get(background.URL) + require.NoError(t, err, "Should send request") resp.Body.Close() - // Save HAR file and verify content - tmpfile := "test.har" - err = logger.SaveToFile(tmpfile) - if err != nil { - t.Fatal(err) - } - defer os.Remove(tmpfile) + time.Sleep(200 * time.Millisecond) - // Read and verify HAR content - harData, err := os.ReadFile(tmpfile) - if err != nil { - t.Fatal(err) - } + entries := logger.GetEntries() + require.Len(t, entries, 1, "Should have one log entry") - var harLog har.Har - if err := json.Unmarshal(harData, &harLog); err != nil { - t.Fatal(err) - } + // Save to file + tmpDir := t.TempDir() + harFilePath := filepath.Join(tmpDir, "test.har") + err = logger.SaveToFile(harFilePath) + require.NoError(t, err, "Should save HAR file") - // Verify we captured both requests - if len(harLog.Log.Entries) != 2 { - t.Errorf("Expected 2 entries in HAR log, got %d", len(harLog.Log.Entries)) - } + // Verify file contents + harData, err := os.ReadFile(harFilePath) + require.NoError(t, err, "Should read HAR file") - // Verify GET request - if harLog.Log.Entries[0].Request.Method != "GET" { - t.Errorf("Expected GET request, got %s", harLog.Log.Entries[0].Request.Method) + var har Har + err = json.Unmarshal(harData, &har) + require.NoError(t, err, "Should parse HAR JSON") + assert.Len(t, har.Log.Entries, 1, "Saved HAR should have one entry") + assert.Equal(t, "1.2", har.Log.Version, "HAR version should be 1.2") + + // Clear logger + logger.Clear() + entries = logger.GetEntries() + assert.Empty(t, entries, "Should have no entries after clear") +} + +func TestHarLoggerConcurrency(t *testing.T) { + logger := NewLogger() + + background := httptest.NewServer(ConstantHandler("concurrent")) + defer background.Close() + + proxyServer := createTestProxy(logger) + defer proxyServer.Close() + + client := createProxyClient(proxyServer.URL) + + requestCount := 50 + successChan := make(chan bool, requestCount) + + for i := 0; i < requestCount; i++ { + go func() { + resp, err := client.Get(background.URL) + if err != nil { + successChan <- false + return + } + resp.Body.Close() + successChan <- true + }() } - // Verify POST request - if harLog.Log.Entries[1].Request.Method != "POST" { - t.Errorf("Expected POST request, got %s", harLog.Log.Entries[1].Request.Method) + successCount := 0 + for i := 0; i < requestCount; i++ { + if <-successChan { + successCount++ + } } + + time.Sleep(500 * time.Millisecond) + + entries := logger.GetEntries() + assert.Equal(t, successCount, len(entries), "Should log all successful requests") } diff --git a/ext/har/types.go b/ext/har/types.go index ed4fe82d..c5b2345c 100644 --- a/ext/har/types.go +++ b/ext/har/types.go @@ -6,9 +6,10 @@ import ( "bytes" "io" "log" - "net" "net/http" "net/url" + "mime" + "strings" "time" ) @@ -114,38 +115,73 @@ type Request struct { } func ParseRequest(req *http.Request, captureContent bool) *Request { - if req == nil { - return nil - } - harRequest := Request{ - Method: req.Method, - Url: req.URL.String(), - HttpVersion: req.Proto, - Cookies: parseCookies(req.Cookies()), - Headers: parseStringArrMap(req.Header), - QueryString: parseStringArrMap((req.URL.Query())), - BodySize: req.ContentLength, - HeadersSize: calcHeaderSize(req.Header), - } + if req == nil { + log.Printf("ParseRequest: nil request") + return nil + } - if captureContent && (req.Method == "POST" || req.Method == "PUT") { - harRequest.PostData = parsePostData(req) - } + log.Printf("ParseRequest: method=%s, captureContent=%v", req.Method, captureContent) + + harRequest := Request{ + Method: req.Method, + Url: req.URL.String(), + HttpVersion: req.Proto, + Cookies: parseCookies(req.Cookies()), + Headers: parseStringArrMap(req.Header), + QueryString: parseStringArrMap((req.URL.Query())), + BodySize: req.ContentLength, + HeadersSize: -1, + } - return &harRequest -} + if captureContent && (req.Method == "POST" || req.Method == "PUT") { + log.Printf("ParseRequest: creating PostData, hasBody=%v, hasGetBody=%v", + req.Body != nil, req.GetBody != nil) -func (harEntry *Entry) fillIPAddress(req *http.Request) { - host := req.URL.Hostname() - - if ip := net.ParseIP(host); ip != nil { - harEntry.ServerIpAddress = ip.String() + harRequest.PostData = &PostData{ + MimeType: req.Header.Get("Content-Type"), + } + + var body []byte + var err error + + if req.Body != nil { + log.Printf("ParseRequest: reading from Body") + body, err = io.ReadAll(req.Body) + if err != nil { + log.Printf("ParseRequest: error reading Body: %v", err) + } else { + // Restore the body + req.Body = io.NopCloser(bytes.NewBuffer(body)) + harRequest.PostData.Text = string(body) + log.Printf("ParseRequest: successfully read body: %s", string(body)) + } + } + + // If body is still empty and GetBody is available, try that + if len(body) == 0 && req.GetBody != nil { + log.Printf("ParseRequest: trying GetBody") + if bodyReader, err := req.GetBody(); err == nil { + if body, err = io.ReadAll(bodyReader); err == nil { + harRequest.PostData.Text = string(body) + log.Printf("ParseRequest: successfully read from GetBody: %s", string(body)) + } else { + log.Printf("ParseRequest: error reading from GetBody: %v", err) + } + bodyReader.Close() + } else { + log.Printf("ParseRequest: error getting fresh body: %v", err) + } + } } + + return &harRequest } -func calcHeaderSize(header http.Header) int64 { - // Directly return -1 as per HAR specification - return -1 + + +func (entry *Entry) fillIPAddress(req *http.Request) { + host := req.URL.Hostname() + entry.ServerIpAddress = host } func parsePostData(req *http.Request) *PostData { @@ -259,16 +295,16 @@ func ParseResponse(resp *http.Response, captureContent bool) *Response { if len(resp.Status) > 4 { statusText = resp.Status[4:] } - redirectURL := resp.Header.Get("Location") + harResponse := Response{ Status: resp.StatusCode, StatusText: statusText, HttpVersion: resp.Proto, Cookies: parseCookies(resp.Cookies()), Headers: parseStringArrMap(resp.Header), - RedirectUrl: redirectURL, + RedirectUrl: resp.Header.Get("Location"), BodySize: resp.ContentLength, - HeadersSize: calcHeaderSize(resp.Header), + HeadersSize: -1, // As per HAR spec } if captureContent && resp.Body != nil { @@ -277,7 +313,6 @@ func ParseResponse(resp *http.Response, captureContent bool) *Response { log.Printf("Error reading response body: %v", err) return &harResponse } - // Create a new reader for the response body resp.Body = io.NopCloser(bytes.NewBuffer(body)) harResponse.Content = Content{ @@ -290,8 +325,6 @@ func ParseResponse(resp *http.Response, captureContent bool) *Response { return &harResponse } - - type Cookie struct { Name string `json:"name"` Value string `json:"value"` From 6bf72e009167f27ac137368880a3508e0a80d305 Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Wed, 1 Jan 2025 17:11:44 +1000 Subject: [PATCH 5/7] added testify dependancy --- ext/go.mod | 7 +++++++ ext/go.sum | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/ext/go.mod b/ext/go.mod index 9c7d3945..feb9872f 100644 --- a/ext/go.mod +++ b/ext/go.mod @@ -4,6 +4,13 @@ go 1.20 require ( github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c + github.com/stretchr/testify v1.10.0 golang.org/x/net v0.33.0 golang.org/x/text v0.21.0 ) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/ext/go.sum b/ext/go.sum index b9cadd46..ce723c58 100644 --- a/ext/go.sum +++ b/ext/go.sum @@ -1,6 +1,16 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c h1:yWAGp1CjD1mQGLUsADqPn5s1n2AkGAX33XLDUgoXzyo= github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c/go.mod h1:P73liMk9TZCyF9fXG/RyMeSizmATvpvy3ZS61/1eXn4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 7439ab3fd4ec9c97b3ee77abdad27c628b02bbf0 Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Wed, 1 Jan 2025 17:27:31 +1000 Subject: [PATCH 6/7] modernised fillIpaddress func to work for ipv6 addresses --- ext/har/types.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/ext/har/types.go b/ext/har/types.go index c5b2345c..774b2520 100644 --- a/ext/har/types.go +++ b/ext/har/types.go @@ -181,7 +181,40 @@ func ParseRequest(req *http.Request, captureContent bool) *Request { func (entry *Entry) fillIPAddress(req *http.Request) { host := req.URL.Hostname() - entry.ServerIpAddress = host + + // try to parse the host as an IP address + if ip := net.ParseIP(host); ip != nil { + entry.ServerIpAddress = ip.String() + return + } + + // If it's not an IP address, perform a DNS lookup with a timeout + resolver := &net.Resolver{} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + ips, err := resolver.LookupIP(ctx, "ip", host) + if err != nil { + // If lookup fails, just use the hostname + entry.ServerIpAddress = host + return + } + + // Prefer IPv4, but fall back to IPv6 if necessary + for _, ip := range ips { + if ipv4 := ip.To4(); ipv4 != nil { + entry.ServerIpAddress = ipv4.String() + return + } + } + + // If no IPv4 address found, use the first IP (IPv6) in the list + if len(ips) > 0 { + entry.ServerIpAddress = ips[0].String() + } else { + // If no IPs found, fall back to the hostname + entry.ServerIpAddress = host + } } func parsePostData(req *http.Request) *PostData { From f1879d22c072c9307f2daa7c289f1b4decd95fef Mon Sep 17 00:00:00 2001 From: CameronBadman Date: Wed, 1 Jan 2025 23:06:27 +1000 Subject: [PATCH 7/7] refactored HAR exporting componant --- ext/har/logger.go | 219 ++++++++++++++++++++++++++++------------- ext/har/logger_test.go | 94 +++++++++++++----- ext/har/types.go | 39 ++++---- 3 files changed, 235 insertions(+), 117 deletions(-) diff --git a/ext/har/logger.go b/ext/har/logger.go index b88023b9..10cebb35 100644 --- a/ext/har/logger.go +++ b/ext/har/logger.go @@ -1,106 +1,185 @@ package har - import ( "encoding/json" "net/http" "os" "sync" "time" - "github.com/elazarl/goproxy" ) +// ExportFunc is a function type that users can implement to handle exported entries +type ExportFunc func([]Entry) + // Logger implements a HAR logging extension for goproxy type Logger struct { - mu sync.Mutex - har *Har - captureContent bool + mu sync.Mutex + entries []Entry + captureContent bool + exportFunc ExportFunc + exportInterval time.Duration + exportCount int + currentCount int + lastExport time.Time + stopChan chan struct{} +} + +// LoggerOption is a function type for configuring the Logger +type LoggerOption func(*Logger) + +// WithExportInterval sets the time interval for exporting entries +func WithExportInterval(d time.Duration) LoggerOption { + return func(l *Logger) { + l.exportInterval = d + } +} + +// WithExportCount sets the number of requests after which to export entries +func WithExportCount(count int) LoggerOption { + return func(l *Logger) { + l.exportCount = count + } } // NewLogger creates a new HAR logger instance -func NewLogger() *Logger { - return &Logger{ - har: New(), - captureContent: true, - } +func NewLogger(exportFunc ExportFunc, opts ...LoggerOption) *Logger { + l := &Logger{ + entries: make([]Entry, 0), + captureContent: true, + exportFunc: exportFunc, + stopChan: make(chan struct{}), + } + + for _, opt := range opts { + opt(l) + } + + go l.exportLoop() + + return l } // OnRequest handles incoming HTTP requests func (l *Logger) OnRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { - // Store the start time in context for later use - if ctx != nil { - ctx.UserData = time.Now() - } - return req, nil -} + if ctx != nil { + ctx.UserData = time.Now() + } + return req, nil +} // OnResponse handles HTTP responses func (l *Logger) OnResponse(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { - if resp == nil || ctx == nil || ctx.Req == nil || ctx.UserData == nil { - return resp - } - - startTime, ok := ctx.UserData.(time.Time) - if !ok { - return resp - } - - // Create HAR entry - entry := Entry{ - StartedDateTime: startTime, - Time: time.Since(startTime).Milliseconds(), - Request: ParseRequest(ctx.Req, l.captureContent), - Response: ParseResponse(resp, l.captureContent), - Timings: Timings{ - Send: 0, - Wait: time.Since(startTime).Milliseconds(), - Receive: 0, - }, - } - - // Add server IP - entry.fillIPAddress(ctx.Req) - - // Add to HAR log thread-safely - l.mu.Lock() - l.har.AppendEntry(entry) - l.mu.Unlock() - - return resp + if resp == nil || ctx == nil || ctx.Req == nil || ctx.UserData == nil { + return resp + } + startTime, ok := ctx.UserData.(time.Time) + if !ok { + return resp + } + + entry := Entry{ + StartedDateTime: startTime, + Time: time.Since(startTime).Milliseconds(), + Request: ParseRequest(ctx.Req, l.captureContent), + Response: ParseResponse(resp, l.captureContent), + Timings: Timings{ + Send: 0, + Wait: time.Since(startTime).Milliseconds(), + Receive: 0, + }, + } + entry.fillIPAddress(ctx.Req) + + l.mu.Lock() + l.entries = append(l.entries, entry) + l.currentCount++ + l.mu.Unlock() + + return resp +} + +func (l *Logger) exportLoop() { + ticker := time.NewTicker(100 * time.Millisecond) // Check frequently + defer ticker.Stop() + + for { + select { + case <-ticker.C: + l.checkAndExport() + case <-l.stopChan: + return + } + } +} + +func (l *Logger) checkAndExport() { + l.mu.Lock() + defer l.mu.Unlock() + + shouldExport := false + if l.exportCount > 0 && l.currentCount >= l.exportCount { + shouldExport = true + } else if l.exportInterval > 0 && time.Since(l.lastExport) >= l.exportInterval { + shouldExport = true + } + + if shouldExport && len(l.entries) > 0 { + l.exportFunc(l.entries) + l.entries = make([]Entry, 0) + l.currentCount = 0 + l.lastExport = time.Now() + } +} + +// Stop stops the export loop +func (l *Logger) Stop() { + close(l.stopChan) } // SaveToFile writes the current HAR log to a file func (l *Logger) SaveToFile(filename string) error { - l.mu.Lock() - defer l.mu.Unlock() - file, err := os.Create(filename) - if err != nil { - return err - } - defer file.Close() - - jsonData, err := json.Marshal(l.har) - if err != nil { - return err - } - - _, err = file.Write(jsonData) - return err + l.mu.Lock() + defer l.mu.Unlock() + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + har := &Har{ + Log: Log{ + Version: "1.2", + Creator: Creator{ + Name: "GoProxy", + Version: "1.0", + }, + Entries: l.entries, + }, + } + + jsonData, err := json.Marshal(har) + if err != nil { + return err + } + + _, err = file.Write(jsonData) + return err } // Clear resets the HAR log func (l *Logger) Clear() { - l.mu.Lock() - defer l.mu.Unlock() - l.har = New() + l.mu.Lock() + defer l.mu.Unlock() + l.entries = make([]Entry, 0) + l.currentCount = 0 } // GetEntries returns a copy of the current HAR entries func (l *Logger) GetEntries() []Entry { - l.mu.Lock() - defer l.mu.Unlock() - entries := make([]Entry, len(l.har.Log.Entries)) - copy(entries, l.har.Log.Entries) - return entries + l.mu.Lock() + defer l.mu.Unlock() + entries := make([]Entry, len(l.entries)) + copy(entries, l.entries) + return entries } diff --git a/ext/har/logger_test.go b/ext/har/logger_test.go index 1f3fd919..72cef0f3 100644 --- a/ext/har/logger_test.go +++ b/ext/har/logger_test.go @@ -42,7 +42,6 @@ func createProxyClient(proxyURL string) *http.Client { return &http.Client{Transport: tr} } - func TestHarLoggerBasicFunctionality(t *testing.T) { testCases := []struct { name string @@ -70,7 +69,13 @@ func TestHarLoggerBasicFunctionality(t *testing.T) { background := httptest.NewServer(ConstantHandler("hello world")) defer background.Close() - logger := NewLogger() + var exportedEntries []Entry + exportFunc := func(entries []Entry) { + exportedEntries = append(exportedEntries, entries...) + } + logger := NewLogger(exportFunc) + defer logger.Stop() + proxyServer := createTestProxy(logger) defer proxyServer.Close() @@ -100,6 +105,9 @@ func TestHarLoggerBasicFunctionality(t *testing.T) { require.Len(t, entries, 1, "Should have one log entry") entry := entries[0] assert.Equal(t, tc.expectedMethod, entry.Request.Method, "Request method should match") + + // Verify exported entries + assert.Len(t, exportedEntries, 0, "Should not have exported entries yet") }) } } @@ -111,7 +119,12 @@ func TestHarLoggerHeaders(t *testing.T) { })) defer background.Close() - logger := NewLogger() + var exportedEntries []Entry + exportFunc := func(entries []Entry) { + exportedEntries = append(exportedEntries, entries...) + } + logger := NewLogger(exportFunc) + defer logger.Stop() proxyServer := createTestProxy(logger) defer proxyServer.Close() @@ -147,7 +160,12 @@ func TestHarLoggerHeaders(t *testing.T) { } func TestHarLoggerSaveAndClear(t *testing.T) { - logger := NewLogger() + var exportedEntries []Entry + exportFunc := func(entries []Entry) { + exportedEntries = append(exportedEntries, entries...) + } + logger := NewLogger(exportFunc) + defer logger.Stop() background := httptest.NewServer(ConstantHandler("test")) defer background.Close() @@ -188,10 +206,15 @@ func TestHarLoggerSaveAndClear(t *testing.T) { assert.Empty(t, entries, "Should have no entries after clear") } -func TestHarLoggerConcurrency(t *testing.T) { - logger := NewLogger() +func TestHarLoggerExportInterval(t *testing.T) { + var exportedEntries []Entry + exportFunc := func(entries []Entry) { + exportedEntries = append(exportedEntries, entries...) + } + logger := NewLogger(exportFunc, WithExportInterval(500*time.Millisecond)) + defer logger.Stop() - background := httptest.NewServer(ConstantHandler("concurrent")) + background := httptest.NewServer(ConstantHandler("test")) defer background.Close() proxyServer := createTestProxy(logger) @@ -199,30 +222,47 @@ func TestHarLoggerConcurrency(t *testing.T) { client := createProxyClient(proxyServer.URL) - requestCount := 50 - successChan := make(chan bool, requestCount) + // Send 3 requests + for i := 0; i < 3; i++ { + resp, err := client.Get(background.URL) + require.NoError(t, err, "Should send request") + resp.Body.Close() + time.Sleep(200 * time.Millisecond) + } + + // Wait for export interval + time.Sleep(600 * time.Millisecond) - for i := 0; i < requestCount; i++ { - go func() { - resp, err := client.Get(background.URL) - if err != nil { - successChan <- false - return - } - resp.Body.Close() - successChan <- true - }() + assert.Len(t, exportedEntries, 3, "Should have exported 3 entries") + assert.Len(t, logger.GetEntries(), 0, "Logger should have no entries after export") +} + +func TestHarLoggerExportCount(t *testing.T) { + var exportedEntries []Entry + exportFunc := func(entries []Entry) { + exportedEntries = append(exportedEntries, entries...) } + logger := NewLogger(exportFunc, WithExportCount(2)) + defer logger.Stop() + + background := httptest.NewServer(ConstantHandler("test")) + defer background.Close() - successCount := 0 - for i := 0; i < requestCount; i++ { - if <-successChan { - successCount++ - } + proxyServer := createTestProxy(logger) + defer proxyServer.Close() + + client := createProxyClient(proxyServer.URL) + + // Send 3 requests + for i := 0; i < 3; i++ { + resp, err := client.Get(background.URL) + require.NoError(t, err, "Should send request") + resp.Body.Close() + time.Sleep(100 * time.Millisecond) } - time.Sleep(500 * time.Millisecond) + time.Sleep(200 * time.Millisecond) - entries := logger.GetEntries() - assert.Equal(t, successCount, len(entries), "Should log all successful requests") + assert.Len(t, exportedEntries, 2, "Should have exported 2 entries") + assert.Len(t, logger.GetEntries(), 1, "Should have 1 entry remaining in logger") } diff --git a/ext/har/types.go b/ext/har/types.go index 774b2520..fbe360ae 100644 --- a/ext/har/types.go +++ b/ext/har/types.go @@ -9,7 +9,8 @@ import ( "net/http" "net/url" "mime" - + "net" + "context" "strings" "time" ) @@ -128,7 +129,7 @@ func ParseRequest(req *http.Request, captureContent bool) *Request { HttpVersion: req.Proto, Cookies: parseCookies(req.Cookies()), Headers: parseStringArrMap(req.Header), - QueryString: parseStringArrMap((req.URL.Query())), + QueryString: parseStringArrMap(req.URL.Query()), BodySize: req.ContentLength, HeadersSize: -1, } @@ -144,7 +145,22 @@ func ParseRequest(req *http.Request, captureContent bool) *Request { var body []byte var err error - if req.Body != nil { + if req.GetBody != nil { + log.Printf("ParseRequest: using GetBody") + bodyReader, err := req.GetBody() + if err == nil { + body, err = io.ReadAll(bodyReader) + if err != nil { + log.Printf("ParseRequest: error reading from GetBody: %v", err) + } else { + harRequest.PostData.Text = string(body) + log.Printf("ParseRequest: successfully read from GetBody: %s", string(body)) + } + bodyReader.Close() + } else { + log.Printf("ParseRequest: error getting fresh body: %v", err) + } + } else if req.Body != nil { log.Printf("ParseRequest: reading from Body") body, err = io.ReadAll(req.Body) if err != nil { @@ -156,29 +172,12 @@ func ParseRequest(req *http.Request, captureContent bool) *Request { log.Printf("ParseRequest: successfully read body: %s", string(body)) } } - - // If body is still empty and GetBody is available, try that - if len(body) == 0 && req.GetBody != nil { - log.Printf("ParseRequest: trying GetBody") - if bodyReader, err := req.GetBody(); err == nil { - if body, err = io.ReadAll(bodyReader); err == nil { - harRequest.PostData.Text = string(body) - log.Printf("ParseRequest: successfully read from GetBody: %s", string(body)) - } else { - log.Printf("ParseRequest: error reading from GetBody: %v", err) - } - bodyReader.Close() - } else { - log.Printf("ParseRequest: error getting fresh body: %v", err) - } - } } return &harRequest } - func (entry *Entry) fillIPAddress(req *http.Request) { host := req.URL.Hostname()