Skip to content

Commit

Permalink
add chrome/ff-specific headers
Browse files Browse the repository at this point in the history
  • Loading branch information
@davemolk
davemolk committed Jan 3, 2023
1 parent cac9e80 commit f0ac8e3
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 19 deletions.
4 changes: 4 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ go install github.com/davemolk/dorking/cmd/dorking@latest

* While Yahoo has a special query system (v*_vt, for instance), just using p seems to work, so I stuck with that.

* Avoiding bot detection...each request gets a randomly assigned user agent corresponding to your os as well as appropriate headers (50/50 chance of chrome or firefox). That being said, Go unfortunately doesn't preserve header order, so if that's important to you, you're going to have to look elsewhere.


## Flags
I decided to keep these as close to what you'd enter into a search bar as possible.
Expand All @@ -62,6 +64,8 @@ Usage of dorking:
site/domain to exclude
-or string
OR term(s)
-os string
operating system (used to spoof user agent)
-q string
search query
-site string
Expand Down
2 changes: 2 additions & 0 deletions cmd/dorking/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type config struct {
not string
notsite string
or string
os string
query string
site string
timeout int
Expand Down Expand Up @@ -48,6 +49,7 @@ func main() {
flag.StringVar(&config.notsite, "notsite", "", "site/domain to exclude")
flag.StringVar(&config.not, "not", "", "term(s) to exclude")
flag.StringVar(&config.or, "or", "", "OR term(s)")
flag.StringVar(&config.os, "os", "w", "operating system (w or m)")
flag.StringVar(&config.query, "q", "", "search query")
flag.StringVar(&config.site, "site", "", "site/domain to search")
flag.IntVar(&config.timeout, "t", 5000, "timeout for request")
Expand Down
5 changes: 5 additions & 0 deletions cmd/dorking/queries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type queryData struct {
contains string
feed string
filetype string
host string
inbody string
intitle string
inurl string
Expand All @@ -33,6 +34,7 @@ func (d *dorking) getQueryData() []queryData {
contains: "contains%3A",
feed: "feed%3A",
filetype: "filetype%3A",
host: "www.bing.com",
inbody: "inbody%3A",
intitle: "intitle%3A",
inurl: "inanchor%3A",
Expand All @@ -50,6 +52,7 @@ func (d *dorking) getQueryData() []queryData {
base: "https://search.brave.com/search?q=",
feed: "feed%3A",
filetype: "filetype%3A",
host: "search.brave.com",
inbody: "inbody%3A",
intitle: "intitle%3A",
inurl: "inurl%3A",
Expand All @@ -66,6 +69,7 @@ func (d *dorking) getQueryData() []queryData {
base: "https://html.duckduckgo.com/html?q=",
feed: "feed%3A",
filetype: "filetype%3A",
host: "duckduckgo.com",
inbody: "inbody%3A",
intitle: "intitle%3A",
inurl: "inurl%3A",
Expand All @@ -83,6 +87,7 @@ func (d *dorking) getQueryData() []queryData {
base: "https://search.yahoo.com/search?p=",
feed: "feed%3A",
filetype: "filetype%3A",
host: "search.yahoo.com",
inbody: "inbody%3A",
intitle: "intitle%3A",
inurl: "inanchor%3A",
Expand Down
112 changes: 93 additions & 19 deletions cmd/dorking/requests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ func (d *dorking) makeRequest(url string) (*bytes.Buffer, error) {
return nil, fmt.Errorf("couldn't make request for %s: %v", url, err)
}

uAgent := d.randomUA()
req.Header.Set("User-Agent", uAgent)
req = d.headers(req)

resp, err := http.DefaultClient.Do(req)
if err != nil {
Expand All @@ -50,25 +49,100 @@ func (d *dorking) makeRequest(url string) (*bytes.Buffer, error) {
return buf, nil
}

// randomUA picks a random user agent obtained from getUA and returns it.
func (d *dorking) randomUA() string {
userAgents := d.getUA()
func (d *dorking) headers(r *http.Request) *http.Request {
if rand.Intn(2) == 1 {
return d.ff(r)
}
return d.chrome(r)
}

func (d *dorking) ff(r *http.Request) *http.Request {
uAgent := d.ffUA()
r.Header.Set("Host", r.URL.Host)
r.Header.Set("User-Agent", uAgent)
r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
r.Header.Set("Accept-Language", "en-US,en;q=0.5")
r.Header.Set("Accept-Encoding", "gzip, deflate, br")
r.Header.Set("DNT", "1")
r.Header.Set("Connection", "keep-alive")
r.Header.Set("Upgrade-Insecure-Requests", "1")
r.Header.Set("Sec-Fetch-Dest", "document")
r.Header.Set("Sec-Fetch-Mode", "navigate")
r.Header.Set("Sec-Fetch-Site", "none")
r.Header.Set("Sec-Fetch-User", "?1")
r.Header.Set("Sec-GCP", "1")
return r
}

func (d *dorking) chrome(r *http.Request) *http.Request {
uAgent := d.chromeUA()
r.Header.Set("Host", r.URL.Host)
r.Header.Set("Connection", "keep-alive")
r.Header.Set("Cache-Control", "max-age=0")
r.Header.Set("sec-ch-ua", `" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"`)
r.Header.Set("sec-ch-ua-mobile", "?0")
switch d.config.os {
case "m":
r.Header.Set("sec-ch-ua-platform", "Macintosh")
default:
r.Header.Set("sec-ch-ua-platform", "Windows")
}
r.Header.Set("Upgrade-Insecure-Requests", "1")
r.Header.Set("User-Agent", uAgent)
r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
r.Header.Set("Sec-Fetch-Site", "none")
r.Header.Set("Sec-Fetch-Mode", "navigate")
r.Header.Set("Sec-Fetch-User", "?1")
r.Header.Set("Sec-Fetch-Dest", "document")
r.Header.Set("Accept-Encoding", "gzip, deflate, br")
r.Header.Set("Accept-Language", "en-US,en;q=0.5")
return r
}

func (d *dorking) ffUA() string {
var userAgents []string
switch d.config.os {
case "m":
userAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:104.0) Gecko/20100101 Firefox/104.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0",
}
default:
userAgents = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0",
}
}
random := rand.Intn(len(userAgents))
return userAgents[random]
}

// getUA returns a string slice of ten user agents.
func (d *dorking) getUA() []string {
return []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
func (d *dorking) chromeUA() string {
var userAgents []string
switch d.config.os {
case "m":
userAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36",
}
default:
userAgents = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
}
}
}
random := rand.Intn(len(userAgents))
return userAgents[random]
}

0 comments on commit f0ac8e3

Please sign in to comment.