Merge pull request #125 from edoardottt/devel

v1.3.2
edoardottt · Jun 9, 2023 · c2f158a · c2f158a
2 parents cf84ea2 + d9ec9c4
commit c2f158a
Show file tree

Hide file tree

Showing 21 changed files with 778 additions and 423 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -5,7 +5,6 @@ linters:
   enable:
     - asciicheck
     - bodyclose
-    - depguard
     - dogsled
     - dupl
     - errcheck
@@ -59,5 +58,5 @@ issues:
   exclude-rules:
     - path: crawler/colly.go
       text: "should replace loop with `(FinalInfos|FinalErrors|FinalSecrets)"
-    - path: pkg/output/beautify.go
+    - path: pkg/output/banner.go
       text: "should be written without leading space as"
diff --git a/Makefile b/Makefile
@@ -1,8 +1,8 @@
 REPO=github.com/edoardottt/cariddi
 
-fmt:
-	@gofmt -s ./*
-	@echo "Done."
+tidy:
+	@go get -u ./...
+	@go mod tidy -v
 
 remod:
 	@rm -rf go.*
@@ -11,7 +11,7 @@ remod:
 	@echo "Done."
 
 update:
-	@go get -u
+	@go get -u ./...
 	@go mod tidy -v
 	@make unlinux
 	@git pull

diff --git a/README.md b/README.md
@@ -33,10 +33,6 @@
   <a href="https://edoardoottavianelli.it">
     <img src="https://github.com/edoardottt/images/blob/main/cariddi/ask-me-anything.svg" alt="ask me anything" />
   </a>
-  <!-- gobadge -->
-  <a href="https://edoardoottavianelli.it">
-    <img src="https://github.com/edoardottt/images/blob/main/cariddi/gobadge" alt="gobadge" />
-  </a>
   <!-- license GPLv3.0 -->
   <a href="https://github.com/edoardottt/cariddi/blob/master/LICENSE">
     <img src="https://github.com/edoardottt/images/blob/main/cariddi/license-GPL3.svg" alt="license-GPL3" />
@@ -60,7 +56,7 @@
   <a href="#license-">License</a>
 </p>
 
-Preview :bar_chart:
+Preview 📊
 ----------
 
 <!--[![asciicast](https://asciinema.org/a/415989.svg)](https://asciinema.org/a/415989)-->
@@ -73,11 +69,13 @@ Installation 📡
 ----------
 
 ### Using Snap
+
 ```bash
 sudo snap install cariddi
 ```
 
 ### Using Go
+
 ```bash
 go install -v github.com/edoardottt/cariddi/cmd/cariddi@latest
 ```
@@ -101,112 +99,112 @@ You need [Go](https://golang.org/).
   - `git clone https://github.com/edoardottt/cariddi.git`
   - `cd cariddi`
   - `go get ./...`
-  - `.\make.bat windows` (to install)    
+  - `.\make.bat windows` (to install)
   - `.\make.bat unwindows` (to uninstall)
 
 Get Started 🎉
 ----------
 
-`cariddi -h` prints the help in the command line.
-
-*Note*: Don't rely on the CLI output, use always `-ot/-oh` to save the output.
+`cariddi -h` prints the help.
 
 ```
 Usage of cariddi:
   -c int
-    	Concurrency level. (default 20)
+     Concurrency level. (default 20)
   -cache
-    	Use the .cariddi_cache folder as cache.
+     Use the .cariddi_cache folder as cache.
   -d int
-    	Delay between a page crawled and another.
+     Delay between a page crawled and another.
   -debug
-    	Print debug information while crawling.
-  -e	Hunt for juicy endpoints.
+     Print debug information while crawling.
+  -e Hunt for juicy endpoints.
   -ef string
-    	Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
+     Use an external file (txt, one per line) to use custom parameters for endpoints hunting.
   -err
-    	Hunt for errors in websites.
+     Hunt for errors in websites.
   -examples
-    	Print the examples.
+     Print the examples.
   -ext int
-    	Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
-  -h	Print the help.
+     Hunt for juicy file extensions. Integer from 1(juicy) to 7(not juicy).
+  -h Print the help.
   -headers string
-    	Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
+     Use custom headers for each request E.g. -headers "Cookie: auth=yes;;Client: type=2".
   -headersfile string
-    	Read from an external file custom headers (same format of headers flag).
+     Read from an external file custom headers (same format of headers flag).
   -json
-      Print the output as JSON in stdout.
+     Print the output as JSON in stdout.
   -i string
-    	Ignore the URL containing at least one of the elements of this array.
+     Ignore the URL containing at least one of the elements of this array.
   -info
-    	Hunt for useful informations in websites.
+     Hunt for useful informations in websites.
   -intensive
-    	Crawl searching for resources matching 2nd level domain.
+     Crawl searching for resources matching 2nd level domain.
   -it string
-    	Ignore the URL containing at least one of the lines of this file.
+     Ignore the URL containing at least one of the lines of this file.
   -oh string
-    	Write the output into an HTML file.
+     Write the output into an HTML file.
   -ot string
-    	Write the output into a TXT file.
+     Write the output into a TXT file.
   -plain
-    	Print only the results.
+     Print only the results.
   -proxy string
-    	Set a Proxy to be used (http and socks5 supported).
+     Set a Proxy to be used (http and socks5 supported).
   -rua
-    	Use a random browser user agent on every request.
-  -s	Hunt for secrets.
+     Use a random browser user agent on every request.
+  -s Hunt for secrets.
   -sf string
-    	Use an external file (txt, one per line) to use custom regexes for secrets hunting.
+     Use an external file (txt, one per line) to use custom regexes for secrets hunting.
+  -sr
+     Store HTTP responses.
   -t int
-    	Set timeout for the requests. (default 10)
+     Set timeout for the requests. (default 10)
   -ua string
-    	Use a custom User Agent.
+     Use a custom User Agent.
   -version
-    	Print the version.
+     Print the version.
 ```
 
-
 Examples 💡
 ----------
 
-  - `cariddi -version` (Print the version)
-  - `cariddi -h` (Print the help)
-  - `cariddi -examples` (Print the examples)
-  - `cat urls | cariddi -s` (Hunt for secrets)
-  - `cat urls | cariddi -d 2` (2 seconds between a page crawled and another)
-  - `cat urls | cariddi -c 200` (Set the concurrency level to 200)
-  - `cat urls | cariddi -e` (Hunt for juicy endpoints)
-  - `cat urls | cariddi -plain` (Print only useful things)
-  - `cat urls | cariddi -ot target_name` (Results in txt file)
-  - `cat urls | cariddi -oh target_name` (Results in html file)
-  - `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files)
-  - `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints)
-  - `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets)
-  - `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words)
-  - `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file)
-  - `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache)
-  - `cat urls | cariddi -t 5` (Set the timeout for the requests)
-  - `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`)
-  - `cat urls | cariddi -rua` (Use a random browser user agent on every request)
-  - `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy (http and socks5 supported))
-  - `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"`
-  - `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers)
-  - `cat urls | cariddi -err` (Hunt for errors in websites)
-  - `cat urls | cariddi -info` (Hunt for useful informations in websites)
-  - `cat urls | cariddi -debug` (Print debug information while crawling)
-  - `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent)
-  - `cat urls | cariddi -json` (Print the output as JSON in stdout)
-  - `cat urls | cariddi -json | jq .` (Pipe the JSON output into jq)
-
-  - For Windows:
-  	- use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt
-  	- or just `cat urls | cariddi.exe` using PowerShell
-
-  - To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration).
+- `cariddi -version` (Print the version)
+- `cariddi -h` (Print the help)
+- `cariddi -examples` (Print the examples)
+- `cat urls | cariddi -s` (Hunt for secrets)
+- `cat urls | cariddi -d 2` (2 seconds between a page crawled and another)
+- `cat urls | cariddi -c 200` (Set the concurrency level to 200)
+- `cat urls | cariddi -e` (Hunt for juicy endpoints)
+- `cat urls | cariddi -plain` (Print only results)
+- `cat urls | cariddi -ot target_name` (Results in txt file)
+- `cat urls | cariddi -oh target_name` (Results in html file)
+- `cat urls | cariddi -ext 2` (Hunt for juicy (level 2 out of 7) files)
+- `cat urls | cariddi -e -ef endpoints_file` (Hunt for custom endpoints)
+- `cat urls | cariddi -s -sf secrets_file` (Hunt for custom secrets)
+- `cat urls | cariddi -i forum,blog,community,open` (Ignore urls containing these words)
+- `cat urls | cariddi -it ignore_file` (Ignore urls containing at least one line in the input file)
+- `cat urls | cariddi -cache` (Use the .cariddi_cache folder as cache)
+- `cat urls | cariddi -t 5` (Set the timeout for the requests)
+- `cat urls | cariddi -intensive` (Crawl searching also subdomains, same as `*.target.com`)
+- `cat urls | cariddi -rua` (Use a random browser user agent on every request)
+- `cat urls | cariddi -proxy http://127.0.0.1:8080` (Set a Proxy, http and socks5 supported)
+- `cat urls | cariddi -headers "Cookie: auth=admin;type=2;; X-Custom: customHeader"`
+- `cat urls | cariddi -headersfile headers.txt` (Read from an external file custom headers)
+- `cat urls | cariddi -err` (Hunt for errors in websites)
+- `cat urls | cariddi -info` (Hunt for useful informations in websites)
+- `cat urls | cariddi -debug` (Print debug information while crawling)
+- `cat urls | cariddi -ua "Custom User Agent"` (Use a custom User Agent)
+- `cat urls | cariddi -json` (Print the output as JSON in stdout)
+- `cat urls | cariddi -sr` (Store HTTP responses)
+
+- For Windows:
+  - use `powershell.exe -Command "cat urls | .\cariddi.exe"` inside the Command prompt
+  - or just `cat urls | cariddi.exe` using PowerShell
+
+- To integrate cariddi with Burpsuite [make sure to follow these steps](https://github.com/edoardottt/cariddi/wiki/BurpSuite-Integration).
 
 Changelog 📌
 -------
+
 Detailed changes for each release are documented in the [release notes](https://github.com/edoardottt/cariddi/releases).
 
 Contributing 🛠
@@ -215,21 +213,17 @@ Contributing 🛠
 Just open an [issue](https://github.com/edoardottt/cariddi/issues)/[pull request](https://github.com/edoardottt/cariddi/pulls).
 
 Before opening a pull request, download [golangci-lint](https://golangci-lint.run/usage/install/) and run
+
 ```bash
 golangci-lint run
 ```
+
 If there aren't errors, go ahead :)
 
 **Help me building this!**
 
-Special thanks to: [go-colly](http://go-colly.org/), [zricethezav](https://github.com/zricethezav/gitleaks/blob/master/config/default.go), [projectdiscovery](https://github.com/projectdiscovery/nuclei-templates/tree/master/file/keys), [tomnomnom](https://github.com/tomnomnom/gf/tree/master/examples) and [RegexPassive](https://github.com/hahwul/RegexPassive).
+Special thanks to: [go-colly](http://go-colly.org/), [zricethezav](https://github.com/zricethezav/gitleaks/blob/master/config/default.go), [projectdiscovery](https://github.com/projectdiscovery/nuclei-templates/tree/master/file/keys), [tomnomnom](https://github.com/tomnomnom/gf/tree/master/examples), [RegexPassive](https://github.com/hahwul/RegexPassive) and [all the contributors](https://github.com/edoardottt/cariddi/wiki/Contributors).
 
-**To do:**
-
-  - [ ] Add more Tests
-
-  - [ ] Tor support
-
 License 📝
 -------
 

diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go
@@ -44,7 +44,7 @@ func main() {
 
 	// Print version and exit.
 	if flags.Version {
-		output.Beautify()
+		output.Banner()
 		os.Exit(0)
 	}
 
@@ -62,7 +62,7 @@ func main() {
 
 	// If it's possible print the cariddi banner.
 	if !flags.Plain {
-		output.Beautify()
+		output.Banner()
 	}
 
 	// Setup the config according to the flags that were
@@ -86,6 +86,7 @@ func main() {
 		InfoFlag:      flags.Info,
 		Debug:         flags.Debug,
 		UserAgent:     flags.UserAgent,
+		StoreResp:     flags.StoreResp,
 	}
 
 	// Read the targets from standard input.
@@ -126,6 +127,10 @@ func main() {
 		output.HeaderHTML("Results", ResultHTML)
 	}
 
+	if config.StoreResp {
+		fileUtils.CreateIndexOutputFile("index.responses.txt")
+	}
+
 	// Read headers if needed
 	if flags.HeadersFile != "" || flags.Headers != "" {
 		var headersInput string

diff --git a/go.mod b/go.mod
@@ -18,11 +18,11 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
 	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect

diff --git a/go.sum b/go.sum
@@ -30,8 +30,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
+github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
@@ -65,8 +65,9 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=