From f0ac8e3a7ebf34c7506ff9f3b774e7d4a1746013 Mon Sep 17 00:00:00 2001 From: Dave Molk Date: Mon, 2 Jan 2023 19:31:35 -0700 Subject: [PATCH] add chrome/ff-specific headers --- README.md | 4 ++ cmd/dorking/main.go | 2 + cmd/dorking/queries.go | 5 ++ cmd/dorking/requests.go | 112 +++++++++++++++++++++++++++++++++------- 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index cfc7cc4..f5dfb2d 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ go install github.com/davemolk/dorking/cmd/dorking@latest * While Yahoo has a special query system (v*_vt, for instance), just using p seems to work, so I stuck with that. +* Avoiding bot detection...each request gets a randomly assigned user agent corresponding to your os as well as appropriate headers (50/50 chance of chrome or firefox). That being said, Go unfortunately doesn't preserve header order, so if that's important to you, you're going to have to look elsewhere. + ## Flags I decided to keep these as close to what you'd enter into a search bar as possible. @@ -62,6 +64,8 @@ Usage of dorking: site/domain to exclude -or string OR term(s) + -os string + operating system (used to spoof user agent) -q string search query -site string diff --git a/cmd/dorking/main.go b/cmd/dorking/main.go index 931c222..1fcbeeb 100644 --- a/cmd/dorking/main.go +++ b/cmd/dorking/main.go @@ -21,6 +21,7 @@ type config struct { not string notsite string or string + os string query string site string timeout int @@ -48,6 +49,7 @@ func main() { flag.StringVar(&config.notsite, "notsite", "", "site/domain to exclude") flag.StringVar(&config.not, "not", "", "term(s) to exclude") flag.StringVar(&config.or, "or", "", "OR term(s)") + flag.StringVar(&config.os, "os", "w", "operating system (w or m)") flag.StringVar(&config.query, "q", "", "search query") flag.StringVar(&config.site, "site", "", "site/domain to search") flag.IntVar(&config.timeout, "t", 5000, "timeout for request") diff --git a/cmd/dorking/queries.go b/cmd/dorking/queries.go index 218362b..35bf96f 100644 --- a/cmd/dorking/queries.go +++ b/cmd/dorking/queries.go @@ -12,6 +12,7 @@ type queryData struct { contains string feed string filetype string + host string inbody string intitle string inurl string @@ -33,6 +34,7 @@ func (d *dorking) getQueryData() []queryData { contains: "contains%3A", feed: "feed%3A", filetype: "filetype%3A", + host: "www.bing.com", inbody: "inbody%3A", intitle: "intitle%3A", inurl: "inanchor%3A", @@ -50,6 +52,7 @@ func (d *dorking) getQueryData() []queryData { base: "https://search.brave.com/search?q=", feed: "feed%3A", filetype: "filetype%3A", + host: "search.brave.com", inbody: "inbody%3A", intitle: "intitle%3A", inurl: "inurl%3A", @@ -66,6 +69,7 @@ func (d *dorking) getQueryData() []queryData { base: "https://html.duckduckgo.com/html?q=", feed: "feed%3A", filetype: "filetype%3A", + host: "duckduckgo.com", inbody: "inbody%3A", intitle: "intitle%3A", inurl: "inurl%3A", @@ -83,6 +87,7 @@ func (d *dorking) getQueryData() []queryData { base: "https://search.yahoo.com/search?p=", feed: "feed%3A", filetype: "filetype%3A", + host: "search.yahoo.com", inbody: "inbody%3A", intitle: "intitle%3A", inurl: "inanchor%3A", diff --git a/cmd/dorking/requests.go b/cmd/dorking/requests.go index 3b2c661..ebaba4f 100644 --- a/cmd/dorking/requests.go +++ b/cmd/dorking/requests.go @@ -30,8 +30,7 @@ func (d *dorking) makeRequest(url string) (*bytes.Buffer, error) { return nil, fmt.Errorf("couldn't make request for %s: %v", url, err) } - uAgent := d.randomUA() - req.Header.Set("User-Agent", uAgent) + req = d.headers(req) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -50,25 +49,100 @@ func (d *dorking) makeRequest(url string) (*bytes.Buffer, error) { return buf, nil } -// randomUA picks a random user agent obtained from getUA and returns it. -func (d *dorking) randomUA() string { - userAgents := d.getUA() +func (d *dorking) headers(r *http.Request) *http.Request { + if rand.Intn(2) == 1 { + return d.ff(r) + } + return d.chrome(r) +} + +func (d *dorking) ff(r *http.Request) *http.Request { + uAgent := d.ffUA() + r.Header.Set("Host", r.URL.Host) + r.Header.Set("User-Agent", uAgent) + r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + r.Header.Set("Accept-Language", "en-US,en;q=0.5") + r.Header.Set("Accept-Encoding", "gzip, deflate, br") + r.Header.Set("DNT", "1") + r.Header.Set("Connection", "keep-alive") + r.Header.Set("Upgrade-Insecure-Requests", "1") + r.Header.Set("Sec-Fetch-Dest", "document") + r.Header.Set("Sec-Fetch-Mode", "navigate") + r.Header.Set("Sec-Fetch-Site", "none") + r.Header.Set("Sec-Fetch-User", "?1") + r.Header.Set("Sec-GCP", "1") + return r +} + +func (d *dorking) chrome(r *http.Request) *http.Request { + uAgent := d.chromeUA() + r.Header.Set("Host", r.URL.Host) + r.Header.Set("Connection", "keep-alive") + r.Header.Set("Cache-Control", "max-age=0") + r.Header.Set("sec-ch-ua", `" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"`) + r.Header.Set("sec-ch-ua-mobile", "?0") + switch d.config.os { + case "m": + r.Header.Set("sec-ch-ua-platform", "Macintosh") + default: + r.Header.Set("sec-ch-ua-platform", "Windows") + } + r.Header.Set("Upgrade-Insecure-Requests", "1") + r.Header.Set("User-Agent", uAgent) + r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + r.Header.Set("Sec-Fetch-Site", "none") + r.Header.Set("Sec-Fetch-Mode", "navigate") + r.Header.Set("Sec-Fetch-User", "?1") + r.Header.Set("Sec-Fetch-Dest", "document") + r.Header.Set("Accept-Encoding", "gzip, deflate, br") + r.Header.Set("Accept-Language", "en-US,en;q=0.5") + return r +} + +func (d *dorking) ffUA() string { + var userAgents []string + switch d.config.os { + case "m": + userAgents = []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:104.0) Gecko/20100101 Firefox/104.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0", + } + default: + userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0", + } + } random := rand.Intn(len(userAgents)) return userAgents[random] } -// getUA returns a string slice of ten user agents. -func (d *dorking) getUA() []string { - return []string{ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", +func (d *dorking) chromeUA() string { + var userAgents []string + switch d.config.os { + case "m": + userAgents = []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36", + } + default: + userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + } } -} + random := rand.Intn(len(userAgents)) + return userAgents[random] +} \ No newline at end of file