diff --git a/cycletls/client.go b/cycletls/client.go index 8fb1f9a5..51731813 100644 --- a/cycletls/client.go +++ b/cycletls/client.go @@ -1,10 +1,11 @@ package cycletls import ( - http "github.com/Danny-Dasilva/fhttp" - + "crypto/tls" "time" + http "github.com/Danny-Dasilva/fhttp" + "golang.org/x/net/proxy" ) @@ -14,6 +15,7 @@ type Browser struct { UserAgent string Cookies []Cookie InsecureSkipVerify bool + Certificates []tls.Certificate forceHTTP1 bool } @@ -22,7 +24,7 @@ var disabledRedirect = func(req *http.Request, via []*http.Request) error { } func clientBuilder(browser Browser, dialer proxy.ContextDialer, timeout int, disableRedirect bool) http.Client { - //if timeout is not set in call default to 15 + // if timeout is not set in call default to 15 if timeout == 0 { timeout = 15 } @@ -30,7 +32,7 @@ func clientBuilder(browser Browser, dialer proxy.ContextDialer, timeout int, dis Transport: newRoundTripper(browser, dialer), Timeout: time.Duration(timeout) * time.Second, } - //if disableRedirect is set to true httpclient will not redirect + // if disableRedirect is set to true httpclient will not redirect if disableRedirect { client.CheckRedirect = disabledRedirect } diff --git a/cycletls/index.go b/cycletls/index.go index 9070d788..7cfde6e9 100644 --- a/cycletls/index.go +++ b/cycletls/index.go @@ -1,10 +1,9 @@ package cycletls import ( + "crypto/tls" "encoding/json" "flag" - http "github.com/Danny-Dasilva/fhttp" - "github.com/gorilla/websocket" "io" "log" nhttp "net/http" @@ -12,6 +11,9 @@ import ( "os" "runtime" "strings" + + http "github.com/Danny-Dasilva/fhttp" + "github.com/gorilla/websocket" ) // Options sets CycleTLS client options @@ -27,9 +29,12 @@ type Options struct { Timeout int `json:"timeout"` DisableRedirect bool `json:"disableRedirect"` HeaderOrder []string `json:"headerOrder"` - OrderAsProvided bool `json:"orderAsProvided"` //TODO + OrderAsProvided bool `json:"orderAsProvided"` // TODO InsecureSkipVerify bool `json:"insecureSkipVerify"` ForceHTTP1 bool `json:"forceHTTP1"` + + // Allow setting client certificates + Certificates []tls.Certificate `json:"certificates"` } type cycleTLSRequest struct { @@ -72,11 +77,12 @@ type CycleTLS struct { // ready Request func processRequest(request cycleTLSRequest) (result fullRequest) { - var browser = Browser{ + browser := Browser{ JA3: request.Options.Ja3, UserAgent: request.Options.UserAgent, Cookies: request.Options.Cookies, InsecureSkipVerify: request.Options.InsecureSkipVerify, + Certificates: request.Options.Certificates, forceHTTP1: request.Options.ForceHTTP1, } @@ -96,10 +102,10 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { log.Fatal(err) } headerorder := []string{} - //master header order, all your headers will be ordered based on this list and anything extra will be appended to the end - //if your site has any custom headers, see the header order chrome uses and then add those headers to this list + // master header order, all your headers will be ordered based on this list and anything extra will be appended to the end + // if your site has any custom headers, see the header order chrome uses and then add those headers to this list if len(request.Options.HeaderOrder) > 0 { - //lowercase headers + // lowercase headers for _, v := range request.Options.HeaderOrder { lowercasekey := strings.ToLower(v) headerorder = append(headerorder, lowercasekey) @@ -136,7 +142,7 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { } headermap := make(map[string]string) - //TODO: Shorten this + // TODO: Shorten this headerorderkey := []string{} for _, key := range headerorder { for k, v := range request.Options.Headers { @@ -146,22 +152,21 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { headerorderkey = append(headerorderkey, lowercasekey) } } - } headerOrder := parseUserAgent(request.Options.UserAgent).HeaderOrder - //ordering the pseudo headers and our normal headers + // ordering the pseudo headers and our normal headers req.Header = http.Header{ http.HeaderOrderKey: headerorderkey, http.PHeaderOrderKey: headerOrder, } - //set our Host header + // set our Host header u, err := url.Parse(request.Options.URL) if err != nil { panic(err) } - //append our normal headers + // append our normal headers for k, v := range request.Options.Headers { if k != "Content-Length" { req.Header.Set(k, v) @@ -170,7 +175,6 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { req.Header.Set("Host", u.Host) req.Header.Set("user-agent", request.Options.UserAgent) return fullRequest{req: req, client: client, options: request} - } func dispatcher(res fullRequest) (response Response, err error) { @@ -183,7 +187,7 @@ func dispatcher(res fullRequest) (response Response, err error) { headers := make(map[string]string) var cookies []*nhttp.Cookie - return Response{RequestID: res.options.RequestID, Status: parsedError.StatusCode, Body: parsedError.ErrorMsg + "-> \n" + string(err.Error()), Headers: headers, Cookies: cookies, FinalUrl: finalUrl}, nil //normally return error here + return Response{RequestID: res.options.RequestID, Status: parsedError.StatusCode, Body: parsedError.ErrorMsg + "-> \n" + string(err.Error()), Headers: headers, Cookies: cookies, FinalUrl: finalUrl}, nil // normally return error here } defer resp.Body.Close() @@ -195,7 +199,6 @@ func dispatcher(res fullRequest) (response Response, err error) { encoding := resp.Header["Content-Encoding"] content := resp.Header["Content-Type"] bodyBytes, err := io.ReadAll(resp.Body) - if err != nil { log.Print("Parse Bytes" + err.Error()) return response, err @@ -222,15 +225,13 @@ func dispatcher(res fullRequest) (response Response, err error) { Cookies: cookies, FinalUrl: finalUrl, }, nil - } // Queue queues request in worker pool func (client CycleTLS) Queue(URL string, options Options, Method string) { - options.URL = URL options.Method = Method - //TODO add timestamp to request + // TODO add timestamp to request opt := cycleTLSRequest{"Queued Request", options} response := processRequest(opt) client.ReqChan <- response @@ -238,17 +239,16 @@ func (client CycleTLS) Queue(URL string, options Options, Method string) { // Do creates a single request func (client CycleTLS) Do(URL string, options Options, Method string) (response Response, err error) { - options.URL = URL options.Method = Method - // Set default values if not provided - if options.Ja3 == "" { - options.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,18-35-65281-45-17513-27-65037-16-10-11-5-13-0-43-23-51,29-23-24,0" - } - if options.UserAgent == "" { + // Set default values if not provided + if options.Ja3 == "" { + options.Ja3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,18-35-65281-45-17513-27-65037-16-10-11-5-13-0-43-23-51,29-23-24,0" + } + if options.UserAgent == "" { // Mac OS Chrome 121 - options.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" - } + options.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36" + } opt := cycleTLSRequest{"cycleTLSRequest", options} res := processRequest(opt) @@ -272,19 +272,17 @@ func Init(workers ...bool) CycleTLS { return CycleTLS{ReqChan: reqChan, RespChan: respChan} } return CycleTLS{} - } // Close closes channels func (client CycleTLS) Close() { close(client.ReqChan) close(client.RespChan) - } // Worker Pool func workerPool(reqChan chan fullRequest, respChan chan Response) { - //MAX + // MAX for i := 0; i < 100; i++ { go worker(reqChan, respChan) } @@ -339,9 +337,7 @@ func writeSocket(respChan chan Response, c *websocket.Conn) { log.Print("Socket WriteMessage Failed" + err.Error()) continue } - } - } } @@ -358,7 +354,7 @@ func WSEndpoint(w nhttp.ResponseWriter, r *nhttp.Request) { // connection ws, err := upgrader.Upgrade(w, r, nil) if err != nil { - //Golang Received a non-standard request to this port, printing request + // Golang Received a non-standard request to this port, printing request var data map[string]interface{} bodyBytes, err := io.ReadAll(r.Body) if err != nil { @@ -384,11 +380,10 @@ func WSEndpoint(w nhttp.ResponseWriter, r *nhttp.Request) { go workerPool(reqChan, respChan) go readSocket(reqChan, ws) - //run as main thread + // run as main thread writeSocket(respChan, ws) } - } func setupRoutes() { diff --git a/cycletls/roundtripper.go b/cycletls/roundtripper.go index 65a762b2..27a6f0eb 100644 --- a/cycletls/roundtripper.go +++ b/cycletls/roundtripper.go @@ -2,10 +2,10 @@ package cycletls import ( "context" + "crypto/tls" "errors" "fmt" "net" - "strings" "sync" @@ -27,6 +27,7 @@ type roundTripper struct { Cookies []Cookie cachedConnections map[string]net.Conn cachedTransports map[string]http.RoundTripper + Certificates []tls.Certificate dialer proxy.ContextDialer forceHTTP1 bool @@ -40,7 +41,7 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { Value: properties.Value, Path: properties.Path, Domain: properties.Domain, - Expires: properties.JSONExpires.Time, //TODO: scuffed af + Expires: properties.JSONExpires.Time, // TODO: scuffed af RawExpires: properties.RawExpires, MaxAge: properties.MaxAge, HttpOnly: properties.HTTPOnly, @@ -107,7 +108,22 @@ func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net. return nil, err } - conn := utls.UClient(rawConn, &utls.Config{ServerName: host, OmitEmptyPsk: true, InsecureSkipVerify: rt.InsecureSkipVerify}, // MinVersion: tls.VersionTLS10, + // Convert tls.Certificate to utls.Certificate + utlsCerts := make([]utls.Certificate, len(rt.Certificates)) + for i, cert := range rt.Certificates { + utlsCerts[i] = utls.Certificate{ + Certificate: cert.Certificate, + PrivateKey: cert.PrivateKey, + Leaf: cert.Leaf, + } + } + + conn := utls.UClient(rawConn, &utls.Config{ + ServerName: host, + OmitEmptyPsk: true, + InsecureSkipVerify: rt.InsecureSkipVerify, + Certificates: utlsCerts, + }, // MinVersion: tls.VersionTLS10, // MaxVersion: tls.VersionTLS13, utls.HelloCustom) @@ -120,7 +136,7 @@ func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net. _ = conn.Close() if err.Error() == "tls: CurvePreferences includes unsupported curve" { - //fix this + // fix this return nil, fmt.Errorf("conn.Handshake() error for tls 1.3 (please retry request): %+v", err) } return nil, fmt.Errorf("uTlsConn.Handshake() error: %+v", err) @@ -176,7 +192,6 @@ func (rt *roundTripper) CloseIdleConnections() { func newRoundTripper(browser Browser, dialer ...proxy.ContextDialer) http.RoundTripper { if len(dialer) > 0 { - return &roundTripper{ dialer: dialer[0], JA3: browser.JA3, @@ -185,6 +200,7 @@ func newRoundTripper(browser Browser, dialer ...proxy.ContextDialer) http.RoundT cachedTransports: make(map[string]http.RoundTripper), cachedConnections: make(map[string]net.Conn), InsecureSkipVerify: browser.InsecureSkipVerify, + Certificates: browser.Certificates, forceHTTP1: browser.forceHTTP1, } } @@ -197,6 +213,7 @@ func newRoundTripper(browser Browser, dialer ...proxy.ContextDialer) http.RoundT cachedTransports: make(map[string]http.RoundTripper), cachedConnections: make(map[string]net.Conn), InsecureSkipVerify: browser.InsecureSkipVerify, + Certificates: browser.Certificates, forceHTTP1: browser.forceHTTP1, } }