From 9e1e1adf78ddd92fea9af9880e884907894dad90 Mon Sep 17 00:00:00 2001 From: Michael Henriksen Date: Sun, 4 Nov 2018 13:30:59 +0100 Subject: [PATCH] New Golang version --- .gitignore | 7 + CHANGELOG.md | 23 +++ Gopkg.lock | 113 ++++++++++++ Gopkg.toml | 62 +++++++ LICENSE.txt | 21 +++ README.md | 128 +++++++++++++ agents/tcp_port_scanner.go | 54 ++++++ agents/url_logger.go | 28 +++ agents/url_publisher.go | 29 +++ agents/url_requester.go | 102 +++++++++++ agents/url_screenshotter.go | 126 +++++++++++++ agents/util.go | 130 +++++++++++++ build.sh | 59 ++++++ core/banner.go | 8 + core/events.go | 8 + core/log.go | 85 +++++++++ core/options.go | 45 +++++ core/ports.go | 19 ++ core/report.go | 351 ++++++++++++++++++++++++++++++++++++ core/session.go | 228 +++++++++++++++++++++++ core/similarity.go | 28 +++ core/urls.go | 43 +++++ main.go | 201 +++++++++++++++++++++ parsers/nmap.go | 79 ++++++++ parsers/regex.go | 31 ++++ release.sh | 31 ++++ 26 files changed, 2039 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 agents/tcp_port_scanner.go create mode 100644 agents/url_logger.go create mode 100644 agents/url_publisher.go create mode 100644 agents/url_requester.go create mode 100644 agents/url_screenshotter.go create mode 100644 agents/util.go create mode 100755 build.sh create mode 100644 core/banner.go create mode 100644 core/events.go create mode 100644 core/log.go create mode 100644 core/options.go create mode 100644 core/ports.go create mode 100644 core/report.go create mode 100644 core/session.go create mode 100644 core/similarity.go create mode 100644 core/urls.go create mode 100644 main.go create mode 100644 parsers/nmap.go create mode 100644 parsers/regex.go create mode 100755 release.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4128fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +aquatone +vendor/ +build/ +screenshots/ +headers/ +html/ +aquatone_report.html diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dc1fff4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.3.2] + +Complete rewrite and simplification of Aquatone. Now written in Go and focused on reporting and screenshotting. + +### Added +- Extraction of hosts, IPs and URLs from arbitrary data piped to Aquatone +- Parsing of Nmap/Masscan XML files +- Clustering of websites with similar structure in HTML report + +### Removed +- Domain discovery (`aquatone-discover`) +- Domain takeover discovery (`aquatone-takeover`) + +[Unreleased]: https://github.com/michenriksen/aquatone/compare/v1.3.2...HEAD +[1.3.2]: https://github.com/michenriksen/aquatone/compare/v1.3.2 diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..ec4fffe --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,113 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "master" + name = "github.com/asaskevich/EventBus" + packages = ["."] + revision = "d46933a94f05c6657d7b923fcf5ac563ee37ec79" + +[[projects]] + name = "github.com/fatih/color" + packages = ["."] + revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" + version = "v1.7.0" + +[[projects]] + branch = "master" + name = "github.com/lair-framework/go-nmap" + packages = ["."] + revision = "84c21710ccc8b9bc7a6cc4b5863bd27a36836c4f" + +[[projects]] + name = "github.com/mattn/go-colorable" + packages = ["."] + revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" + version = "v0.0.9" + +[[projects]] + name = "github.com/mattn/go-isatty" + packages = ["."] + revision = "6ca4dbf54d38eea1a992b3c722a76a5d1c4cb25c" + version = "v0.0.4" + +[[projects]] + branch = "master" + name = "github.com/moul/http2curl" + packages = ["."] + revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d" + +[[projects]] + name = "github.com/mvdan/xurls" + packages = ["."] + revision = "e52e821cbfe8fe163ff6f8628ab5869b11fc05af" + version = "v2.0.0" + +[[projects]] + name = "github.com/parnurzeal/gorequest" + packages = ["."] + revision = "a578a48e8d6ca8b01a3b18314c43c6716bb5f5a3" + version = "v0.2.15" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/remeh/sizedwaitgroup" + packages = ["."] + revision = "5e7302b12ccef91dce9fde2f5bda6d5c7ea5d2eb" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "html", + "html/atom", + "idna", + "publicsuffix" + ] + revision = "9b4f9f5ad5197c79fd623a3638e70d8b26cef344" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "95b1ffbd15a57cc5abb3f04402b9e8ec0016a52c" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "190885f6a1ee69d745b9b7f2825613807a7cdb94c5b2cdce325d0ae39ccb4706" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..4b3e84d --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,62 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[prune] + go-tests = true + unused-packages = true + +[[constraint]] + branch = "master" + name = "github.com/remeh/sizedwaitgroup" + +[[constraint]] + branch = "master" + name = "github.com/asaskevich/EventBus" + +[[constraint]] + name = "github.com/fatih/color" + version = "1.7.0" + +[[constraint]] + name = "github.com/parnurzeal/gorequest" + version = "0.2.15" + +[[constraint]] + name = "github.com/mvdan/xurls" + version = "2.0.0" + +[[constraint]] + branch = "master" + name = "golang.org/x/net" + +[[constraint]] + name = "github.com/pmezard/go-difflib" + version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/lair-framework/go-nmap" diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..229ae9c --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Michael Henriksen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e538e3e --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# AQUATONE + +Aquatone is a tool for visual inspection of websites across a large amount of hosts and is convenient for quickly gaining an overview of HTTP-based attack surface. + +## Installation + +1. Install [Google Chrome](https://www.google.com/chrome/) or [Chromium](https://www.chromium.org/getting-involved/download-chromium) browser -- **Note:** Google Chrome is currently giving unreliable results when running in *headless* mode, so it is recommended to install Chromium for the best results. +2. Download the [latest release](https://github.com/michenriksen/go-aquatone/releases/latest) of Aquatone for your operating system. +3. Uncompress the zip file and move the `aquatone` binary to your desired location. You probably want to move it to a location in your `$PATH` for easier use. + +### Compiling the source code + +If you for some reason don't trust the pre-compiled binaries, you can also compile the code yourself. **You are on your own if you want to do this. I do not support compiling problems. Good luck with it!** + +## Usage + +### Command-line options: + +``` + -chrome-path string + Full path to the Chrome/Chromium executable to use. By default, aquatone will search for Chrome or Chromium + -debug + Print debugging information + -http-timeout int + Timeout in miliseconds for HTTP requests (default 3000) + -nmap + Parse input as Nmap/Masscan XML + -out string + Directory to write files to (default ".") + -ports string + Ports to scan on hosts. Supported list aliases: small, medium, large, xlarge (default "80,443,8000,8080,8443") + -proxy string + Proxy to use for HTTP requests + -resolution string + screenshot resolution (default "1440,900") + -save-body + Save response bodies to files (default true) + -scan-timeout int + Timeout in miliseconds for port scans (default 100) + -screenshot-timeout int + Timeout in miliseconds for screenshots (default 30000) + -silent + Suppress all output except for errors + -threads int + Number of concurrent threads (default number of logical CPUs) +``` + +### Giving Aquatone data + +Aquatone is designed to be as easy to use as possible and to integrate with your existing toolset with no or minimal glue. Aquatone is started by piping output of a command into the tool. It doesn't really care how the piped data looks as URLs, domains, and IP addresses will be extracted with regular expression pattern matching. This means that you can pretty much give it output of any tool you use for host discovery. + +IPs, hostnames and domain names in the data will undergo scanning for ports that are typically used for web services and transformed to URLs with correct scheme. If the data contains URLs, they are assumed to be alive and do not undergo port scanning. + +**Example:** + + $ cat targets.txt | aquatone + +### Output + +When Aquatone is done processing the target hosts, it has created a bunch of files and folders in the current directory: + + - **aquatone_report.html**: An HTML report to open in a browser that displays all the collected screenshots and response headers clustered by similarity. + - **headers/**: A folder with files containing raw response headers from processed targets + - **html/**: A folder with files containing the raw response bodies from processed targets. If you are processing a large amount of hosts, and don't need this for further analysis, you can disable this with the `-save-body=false` flag to save some disk space. + - **screenshots/**: A folder with PNG screenshots of the processed targets + +The output can easily be zipped up and shared with others or archived. + +#### Changing the output destination + +If you don't want Aquatone to create files in the current working directory, you can specify a different location with the `-out` flag: + + $ cat hosts.txt | aquatone -out ~/aquatone/example.com + +Note: Aquatone requires that the output destination folder exists. + + +### Specifying ports to scan + +Be default, Aquatone will scan target hosts with a small list of commonly used HTTP ports: 80, 443, 8000, 8080 and 8443. You can change this to your own list of ports with the `-ports` flag: + + $ cat hosts.txt | aquatone -ports 80,443,3000,3001 + +Aquatone also supports aliases of built-in port lists to make it easier for you: + + - **small**: 80, 443 + - **medium**: 80, 443, 8000, 8080, 8443 (same as default) + - **large**: 80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001, 8008, 8080, 8083, 8443, 8834, 8888 + - **xlarge**: 80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, 2082, 2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, 4711, 4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, 7474, 8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, 8090, 8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, 8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, 9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092, 20720, 28017 + +**Example:** + + $ cat hosts.txt | aquatone -ports large + + +### Usage examples + +Aquatone is designed to play nicely with all kinds of tools. Here's some examples: + +#### Amass DNS enumeration + +[Amass](https://github.com/OWASP/Amass) is currently my preferred tool for enumerating DNS. It uses a bunch of OSINT sources as well as active brute-forcing and clever permutations to quickly identify hundreds, if not thousands, of subdomains on a domain: + +```bash +$ amass -active -brute -o hosts.txt -d yahoo.com +alerts.yahoo.com +ads.yahoo.com +am.yahoo.com +- - - SNIP - - - +prd-vipui-01.infra.corp.gq1.yahoo.com +cp103.mail.ir2.yahoo.com +prd-vipui-01.infra.corp.bf1.yahoo.com +$ cat hosts.txt | aquatone +``` + +There are plenty of other DNS enumeration tools out there and Aquatone should work just as well with any other tool: + +- [Sublist3r](https://github.com/aboul3la/Sublist3r) +- [Subfinder](https://github.com/subfinder/subfinder) +- [Knock](https://github.com/guelfoweb/knock) +- [Fierce](https://www.aldeid.com/wiki/Fierce) +- [Gobuster](https://github.com/OJ/gobuster) + +#### Nmap or Masscan + +Aquatone can make a report on hosts scanned with the [Nmap](https://nmap.org/) or [Masscan](https://github.com/robertdavidgraham/masscan) portscanner. Simply feed Aquatone the XML output and give it the `-nmap` flag to tell it to parse the input as Nmap/Masscan XML: + + $ cat scan.xml | aquatone -nmap diff --git a/agents/tcp_port_scanner.go b/agents/tcp_port_scanner.go new file mode 100644 index 0000000..6ba4a51 --- /dev/null +++ b/agents/tcp_port_scanner.go @@ -0,0 +1,54 @@ +package agents + +import ( + "fmt" + "net" + "time" + + "github.com/michenriksen/aquatone/core" +) + +type TCPPortScanner struct { + session *core.Session +} + +func NewTCPPortScanner() *TCPPortScanner { + return &TCPPortScanner{} +} + +func (d *TCPPortScanner) ID() string { + return "agent:tcp_port_scanner" +} + +func (a *TCPPortScanner) Register(s *core.Session) error { + s.EventBus.SubscribeAsync(core.Host, a.OnHost, false) + a.session = s + return nil +} + +func (a *TCPPortScanner) OnHost(host string) { + a.session.Out.Debug("[%s] Received new host: %s\n", a.ID(), host) + for _, port := range a.session.Ports { + a.session.WaitGroup.Add() + go func(port int, host string) { + defer a.session.WaitGroup.Done() + if a.scanPort(port, host) { + a.session.Stats.IncrementPortOpen() + a.session.Out.Info("%s port %d: %s\n", host, port, Green("open")) + a.session.EventBus.Publish(core.TCPPort, port, host) + } else { + a.session.Stats.IncrementPortClosed() + a.session.Out.Debug("[%s] Port %d is closed on %s\n", a.ID(), port, host) + } + }(port, host) + } +} + +func (a *TCPPortScanner) scanPort(port int, host string) bool { + conn, _ := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), time.Duration(*a.session.Options.ScanTimeout)*time.Millisecond) + if conn != nil { + conn.Close() + return true + } + return false +} diff --git a/agents/url_logger.go b/agents/url_logger.go new file mode 100644 index 0000000..dff3b0f --- /dev/null +++ b/agents/url_logger.go @@ -0,0 +1,28 @@ +package agents + +import ( + "github.com/michenriksen/aquatone/core" +) + +type URLLogger struct { + session *core.Session +} + +func NewURLLogger() *URLLogger { + return &URLLogger{} +} + +func (d *URLLogger) ID() string { + return "agent:url_logger" +} + +func (a *URLLogger) Register(s *core.Session) error { + s.EventBus.SubscribeAsync(core.URLResponsive, a.OnURLResponsive, false) + a.session = s + return nil +} + +func (a *URLLogger) OnURLResponsive(url string) { + a.session.Out.Debug("[%s] Received new url: %s\n", a.ID(), url) + a.session.AddResponsiveURL(url) +} diff --git a/agents/url_publisher.go b/agents/url_publisher.go new file mode 100644 index 0000000..32f53e8 --- /dev/null +++ b/agents/url_publisher.go @@ -0,0 +1,29 @@ +package agents + +import ( + "github.com/michenriksen/aquatone/core" +) + +type URLPublisher struct { + session *core.Session +} + +func NewURLPublisher() *URLPublisher { + return &URLPublisher{} +} + +func (d *URLPublisher) ID() string { + return "agent:url_publisher" +} + +func (a *URLPublisher) Register(s *core.Session) error { + s.EventBus.SubscribeAsync(core.TCPPort, a.OnTCPPort, false) + a.session = s + return nil +} + +func (a *URLPublisher) OnTCPPort(port int, host string) { + a.session.Out.Debug("[%s] Received new open port on %s: %d\n", a.ID(), host, port) + url := HostAndPortToURL(host, port, "") + a.session.EventBus.Publish(core.URL, url) +} diff --git a/agents/url_requester.go b/agents/url_requester.go new file mode 100644 index 0000000..56a13fe --- /dev/null +++ b/agents/url_requester.go @@ -0,0 +1,102 @@ +package agents + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/michenriksen/aquatone/core" + "github.com/parnurzeal/gorequest" +) + +type URLRequester struct { + session *core.Session +} + +func NewURLRequester() *URLRequester { + return &URLRequester{} +} + +func (d *URLRequester) ID() string { + return "agent:url_requester" +} + +func (a *URLRequester) Register(s *core.Session) error { + s.EventBus.SubscribeAsync(core.URL, a.OnURL, false) + a.session = s + return nil +} + +func (a *URLRequester) OnURL(url string) { + a.session.Out.Debug("[%s] Received new URL %s\n", a.ID(), url) + a.session.WaitGroup.Add() + go func(url string) { + defer a.session.WaitGroup.Done() + http := Gorequest(a.session.Options) + resp, _, errs := http.Get(url).End() + var status string + if errs != nil { + a.session.Stats.IncrementRequestFailed() + for _, err := range errs { + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + if os.IsTimeout(err) { + a.session.Out.Error("%s: request timeout\n", url) + return + } + } + a.session.Out.Debug("%s: failed\n", url) + return + } + + a.session.Stats.IncrementRequestSuccessful() + if resp.StatusCode >= 500 { + a.session.Stats.IncrementResponseCode5xx() + status = Red(resp.Status) + } else if resp.StatusCode >= 400 { + a.session.Stats.IncrementResponseCode4xx() + status = Yellow(resp.Status) + } else if resp.StatusCode >= 300 { + a.session.Stats.IncrementResponseCode3xx() + status = Green(resp.Status) + } else { + a.session.Stats.IncrementResponseCode2xx() + status = Green(resp.Status) + } + a.session.Out.Info("%s: %s\n", url, status) + + a.writeHeaders(url, resp) + if *a.session.Options.SaveBody { + a.writeBody(url, resp) + } + + a.session.EventBus.Publish(core.URLResponsive, url) + }(url) +} + +func (a *URLRequester) writeHeaders(url string, resp gorequest.Response) { + filepath := a.session.GetFilePath(fmt.Sprintf("headers/%s.txt", BaseFilenameFromURL(url))) + headers := fmt.Sprintf("%s\n", resp.Status) + for name, value := range resp.Header { + headers += fmt.Sprintf("%v: %v\n", name, strings.Join(value, " ")) + } + if err := ioutil.WriteFile(filepath, []byte(headers), 0644); err != nil { + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + a.session.Out.Error("Failed to write HTTP response headers for %s to %s\n", url, filepath) + } +} + +func (a *URLRequester) writeBody(url string, resp gorequest.Response) { + filepath := a.session.GetFilePath(fmt.Sprintf("html/%s.html", BaseFilenameFromURL(url))) + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + a.session.Out.Error("Failed to read response body for %s\n", url) + return + } + + if err := ioutil.WriteFile(filepath, body, 0644); err != nil { + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + a.session.Out.Error("Failed to write HTTP response body for %s to %s\n", url, filepath) + } +} diff --git a/agents/url_screenshotter.go b/agents/url_screenshotter.go new file mode 100644 index 0000000..04b7f6d --- /dev/null +++ b/agents/url_screenshotter.go @@ -0,0 +1,126 @@ +package agents + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "github.com/michenriksen/aquatone/core" +) + +type URLScreenshotter struct { + session *core.Session + chromePath string +} + +func NewURLScreenshotter() *URLScreenshotter { + return &URLScreenshotter{} +} + +func (d *URLScreenshotter) ID() string { + return "agent:url_screenshotter" +} + +func (a *URLScreenshotter) Register(s *core.Session) error { + s.EventBus.SubscribeAsync(core.URLResponsive, a.OnURLResponsive, false) + a.session = s + a.locateChrome() + + return nil +} + +func (a *URLScreenshotter) OnURLResponsive(url string) { + a.session.Out.Debug("[%s] Received new responsive URL %s\n", a.ID(), url) + a.session.WaitGroup.Add() + go func(url string) { + defer a.session.WaitGroup.Done() + a.screenshotURL(url) + }(url) +} + +func (a *URLScreenshotter) locateChrome() { + if *a.session.Options.ChromePath != "" { + a.chromePath = *a.session.Options.ChromePath + return + } + + paths := []string{ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-beta", + "/usr/bin/google-chrome-unstable", + "/usr/bin/chromium-browser", + "/usr/bin/chromium", + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe", + } + + for _, path := range paths { + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + a.chromePath = path + } + + if a.chromePath == "" { + a.session.Out.Fatal("Unable to locate a valid installation of Chrome. Install Google Chrome or try specifying a valid location with the -chrome-path option.\n") + os.Exit(1) + } + + if strings.Contains(strings.ToLower(a.chromePath), "chrome") { + a.session.Out.Warn("Using unreliable Google Chrome for screenshots. Install Chromium for better results.\n\n") + } + + a.session.Out.Debug("[%s] Located Chrome/Chromium binary at %s\n", a.ID(), a.chromePath) +} + +func (a *URLScreenshotter) screenshotURL(s string) { + filePath := a.session.GetFilePath(fmt.Sprintf("screenshots/%s.png", BaseFilenameFromURL(s))) + var chromeArguments = []string{ + "--headless", "--disable-gpu", "--hide-scrollbars", "--mute-audio", "--disable-notifications", + "--disable-crash-reporter", + "--ignore-certificate-errors", + "--user-agent=" + RandomUserAgent(), + "--window-size=" + *a.session.Options.Resolution, "--screenshot=" + filePath, + } + + if os.Geteuid() == 0 { + chromeArguments = append(chromeArguments, "--no-sandbox") + } + + if *a.session.Options.Proxy != "" { + chromeArguments = append(chromeArguments, "--proxy-server="+*a.session.Options.Proxy) + } + + chromeArguments = append(chromeArguments, s) + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*a.session.Options.ScreenshotTimeout)*time.Millisecond) + defer cancel() + + cmd := exec.CommandContext(ctx, a.chromePath, chromeArguments...) + if err := cmd.Start(); err != nil { + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + a.session.Stats.IncrementScreenshotFailed() + a.session.Out.Error("%s: screenshot failed: %s\n", s, err) + return + } + + if err := cmd.Wait(); err != nil { + a.session.Stats.IncrementScreenshotFailed() + a.session.Out.Debug("[%s] Error: %v\n", a.ID(), err) + if ctx.Err() == context.DeadlineExceeded { + a.session.Out.Error("%s: screenshot timed out\n", s) + return + } + + a.session.Out.Error("%s: screenshot failed: %s\n", s, err) + return + } + + a.session.Stats.IncrementScreenshotSuccessful() + a.session.Out.Info("%s: %s\n", s, Green("screenshot successful")) +} diff --git a/agents/util.go b/agents/util.go new file mode 100644 index 0000000..01a92e4 --- /dev/null +++ b/agents/util.go @@ -0,0 +1,130 @@ +package agents + +import ( + "crypto/tls" + "fmt" + "math/rand" + "net/url" + "strconv" + "strings" + "time" + + "github.com/michenriksen/aquatone/core" + + "github.com/fatih/color" + "github.com/parnurzeal/gorequest" +) + +var ( + UserAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", + "Mozilla/5.0 (X11; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:63.0) Gecko/20100101 Firefox/63.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/69.0.3497.81 Chrome/69.0.3497.81 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:62.0) Gecko/20100101 Firefox/62.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; rv:60.0) Gecko/20100101 Firefox/60.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1 Safari/605.1.15", + } + green = color.New(color.FgGreen).SprintFunc() + yellow = color.New(color.FgYellow).SprintfFunc() + red = color.New(color.FgRed).SprintFunc() +) + +func RandomUserAgent() string { + return UserAgents[rand.Intn(len(UserAgents))] +} + +func RandomIPv4Address() string { + rand.Seed(time.Now().UnixNano()) + blocks := []string{} + for i := 0; i < 4; i++ { + number := rand.Intn(255) + blocks = append(blocks, strconv.Itoa(number)) + } + + return strings.Join(blocks, ".") +} + +func URLEscape(s string) string { + return url.QueryEscape(s) +} + +func Gorequest(o core.Options) *gorequest.SuperAgent { + return gorequest.New(). + Proxy(*o.Proxy). + Timeout(time.Duration(*o.HTTPTimeout)*time.Millisecond). + SetDebug(*o.Debug). + TLSClientConfig(&tls.Config{InsecureSkipVerify: true}). + Set("User-Agent", RandomUserAgent()). + Set("X-Forwarded-For", RandomIPv4Address()). + Set("Via", fmt.Sprintf("1.1 %s", RandomIPv4Address())). + Set("Forwarded", fmt.Sprintf("for=%s;proto=http;by=%s", RandomIPv4Address(), RandomIPv4Address())) +} + +func BaseFilenameFromURL(s string) string { + u, err := url.Parse(s) + if err != nil { + return "" + } + host := strings.Replace(u.Host, ":", "__", 1) + filename := fmt.Sprintf("%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1)) + return strings.ToLower(filename) +} + +func HostAndPortToURL(host string, port int, protocol string) string { + return core.HostAndPortToURL(host, port, protocol) +} + +func Green(s string) string { + return green(s) +} + +func Yellow(s string) string { + return yellow(s) +} + +func Red(s string) string { + return red(s) +} diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..ea43da0 --- /dev/null +++ b/build.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +BUILD_FOLDER=build +VERSION=$(cat core/banner.go | grep Version | cut -d '"' -f 2) + +bin_dep() { + BIN=$1 + which $BIN > /dev/null || { echo "[-] Dependency $BIN not found !"; exit 1; } +} + +create_exe_archive() { + bin_dep 'zip' + + OUTPUT=$1 + + echo "[*] Creating archive $OUTPUT ..." + zip -j "$OUTPUT" aquatone.exe ../README.md ../LICENSE.txt > /dev/null + rm -rf aquatone aquatone.exe +} + +create_archive() { + bin_dep 'zip' + + OUTPUT=$1 + + echo "[*] Creating archive $OUTPUT ..." + zip -j "$OUTPUT" aquatone ../README.md ../LICENSE.md > /dev/null + rm -rf aquatone aquatone.exe +} + +build_linux_amd64() { + echo "[*] Building linux/amd64 ..." + GOOS=linux GOARCH=amd64 go build -o aquatone .. +} + +build_macos_amd64() { + echo "[*] Building darwin/amd64 ..." + GOOS=darwin GOARCH=amd64 go build -o aquatone .. +} + +build_windows_amd64() { + echo "[*] Building windows/amd64 ..." + GOOS=windows GOARCH=amd64 go build -o aquatone.exe .. +} + +rm -rf $BUILD_FOLDER +mkdir $BUILD_FOLDER +cd $BUILD_FOLDER + +build_linux_amd64 && create_archive aquatone_linux_amd64_$VERSION.zip +build_macos_amd64 && create_archive aquatone_macos_amd64_$VERSION.zip +build_windows_amd64 && create_exe_archive aquatone_windows_amd64_$VERSION.zip +shasum -a 256 * > checksums.txt + +echo +echo +du -sh * + +cd -- diff --git a/core/banner.go b/core/banner.go new file mode 100644 index 0000000..4b4e5ae --- /dev/null +++ b/core/banner.go @@ -0,0 +1,8 @@ +package core + +const ( + Name = "aquatone" + Version = "1.3.2" + Author = "Michael Henriksen" + Website = "https://github.com/michenriksen/aquatone" +) diff --git a/core/events.go b/core/events.go new file mode 100644 index 0000000..ca310ec --- /dev/null +++ b/core/events.go @@ -0,0 +1,8 @@ +package core + +const ( + Host = "host" + URL = "url" + URLResponsive = "url:responsive" + TCPPort = "port:tcp" +) diff --git a/core/log.go b/core/log.go new file mode 100644 index 0000000..9f21bc9 --- /dev/null +++ b/core/log.go @@ -0,0 +1,85 @@ +package core + +import ( + "fmt" + "os" + "sync" + + "github.com/fatih/color" +) + +const ( + FATAL = 5 + ERROR = 4 + WARN = 3 + IMPORTANT = 2 + INFO = 1 + DEBUG = 0 +) + +var LogColors = map[int]*color.Color{ + FATAL: color.New(color.FgRed).Add(color.Bold), + ERROR: color.New(color.FgRed), + WARN: color.New(color.FgYellow), + IMPORTANT: color.New(color.Bold), + DEBUG: color.New(color.FgCyan).Add(color.Faint), +} + +type Logger struct { + sync.Mutex + + debug bool + silent bool +} + +func (l *Logger) SetSilent(s bool) { + l.silent = s +} + +func (l *Logger) SetDebug(d bool) { + l.debug = d +} + +func (l *Logger) Log(level int, format string, args ...interface{}) { + l.Lock() + defer l.Unlock() + if level == DEBUG && !l.debug { + return + } else if level < ERROR && l.silent { + return + } + + if c, ok := LogColors[level]; ok { + c.Printf(format, args...) + } else { + fmt.Printf(format, args...) + } + + if level == FATAL { + os.Exit(1) + } +} + +func (l *Logger) Fatal(format string, args ...interface{}) { + l.Log(FATAL, format, args...) +} + +func (l *Logger) Error(format string, args ...interface{}) { + l.Log(ERROR, format, args...) +} + +func (l *Logger) Warn(format string, args ...interface{}) { + l.Log(WARN, format, args...) +} + +func (l *Logger) Important(format string, args ...interface{}) { + l.Log(IMPORTANT, format, args...) +} + +func (l *Logger) Info(format string, args ...interface{}) { + l.Log(INFO, format, args...) +} + +func (l *Logger) Debug(format string, args ...interface{}) { + l.Log(DEBUG, format, args...) +} diff --git a/core/options.go b/core/options.go new file mode 100644 index 0000000..7fc8643 --- /dev/null +++ b/core/options.go @@ -0,0 +1,45 @@ +package core + +import ( + "flag" + "fmt" + "strings" +) + +type Options struct { + Threads *int + OutDir *string + Proxy *string + ChromePath *string + Resolution *string + Ports *string + ScanTimeout *int + HTTPTimeout *int + ScreenshotTimeout *int + Nmap *bool + SaveBody *bool + Silent *bool + Debug *bool +} + +func ParseOptions() (Options, error) { + options := Options{ + Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), + OutDir: flag.String("out", ".", "Directory to write files to"), + Proxy: flag.String("proxy", "", "Proxy to use for HTTP requests"), + ChromePath: flag.String("chrome-path", "", "Full path to the Chrome/Chromium executable to use. By default, aquatone will search for Chrome or Chromium"), + Resolution: flag.String("resolution", "1440,900", "screenshot resolution"), + Ports: flag.String("ports", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(MediumPortList)), ","), "[]"), "Ports to scan on hosts. Supported list aliases: small, medium, large, xlarge"), + ScanTimeout: flag.Int("scan-timeout", 100, "Timeout in miliseconds for port scans"), + HTTPTimeout: flag.Int("http-timeout", 3*1000, "Timeout in miliseconds for HTTP requests"), + ScreenshotTimeout: flag.Int("screenshot-timeout", 30*1000, "Timeout in miliseconds for screenshots"), + Nmap: flag.Bool("nmap", false, "Parse input as Nmap/Masscan XML"), + SaveBody: flag.Bool("save-body", true, "Save response bodies to files"), + Silent: flag.Bool("silent", false, "Suppress all output except for errors"), + Debug: flag.Bool("debug", false, "Print debugging information"), + } + + flag.Parse() + + return options, nil +} diff --git a/core/ports.go b/core/ports.go new file mode 100644 index 0000000..ef6f103 --- /dev/null +++ b/core/ports.go @@ -0,0 +1,19 @@ +package core + +var ( + SmallPortList = []int{80, 443} + + MediumPortList = []int{80, 443, 8000, 8080, 8443} + + LargePortList = []int{80, 81, 443, 591, 2082, 2087, 2095, 2096, 3000, 8000, 8001, + 8008, 8080, 8083, 8443, 8834, 8888} + + XLargePortList = []int{80, 81, 300, 443, 591, 593, 832, 981, 1010, 1311, + 2082, 2087, 2095, 2096, 2480, 3000, 3128, 3333, 4243, 4567, + 4711, 4712, 4993, 5000, 5104, 5108, 5800, 6543, 7000, 7396, + 7474, 8000, 8001, 8008, 8014, 8042, 8069, 8080, 8081, 8088, + 8090, 8091, 8118, 8123, 8172, 8222, 8243, 8280, 8281, 8333, + 8443, 8500, 8834, 8880, 8888, 8983, 9000, 9043, 9060, 9080, + 9090, 9091, 9200, 9443, 9800, 9981, 12443, 16080, 18091, 18092, + 20720, 28017} +) diff --git a/core/report.go b/core/report.go new file mode 100644 index 0000000..59eacb3 --- /dev/null +++ b/core/report.go @@ -0,0 +1,351 @@ +package core + +import ( + "fmt" + "html/template" + "io" + "os" + "strings" +) + +type Header struct { + Name string + Value string +} + +func (h Header) IsInsecure() bool { + switch strings.ToLower(h.Name) { + case "server", "x-powered-by": + return true + case "access-control-allow-origin": + if h.Value == "*" { + return true + } + case "x-xss-protection": + if !strings.HasPrefix(h.Value, "1") { + return true + } + } + return false +} + +func (h Header) IsSecure() bool { + switch strings.ToLower(h.Name) { + case "content-security-policy", "content-security-policy-report-only": + return true + case "strict-transport-security": + return true + case "x-frame-options": + return true + case "referrer-policy": + return true + case "public-key-pins": + return true + case "x-permitted-cross-domain-policies": + if strings.ToLower(h.Value) == "master-only" { + return true + } + case "x-content-type-options": + if strings.ToLower(h.Value) == "nosniff" { + return true + } + case "x-xss-protection": + if strings.HasPrefix(h.Value, "1") { + return true + } + } + return false +} + +type Page struct { + URL string + Status string + Headers []Header + HeadersPath string + BodyPath string + ScreenshotPath string + HasScreenshot bool +} + +const ( + Template = ` + + + + + + + AQUATONE REPORT + + + + + {{range .Clusters}} +
+ {{range .}} +
+
+
{{.URL}}
+
{{.Status}}
+
+ {{if .HasScreenshot}} +
+ +
+ {{else}} + + {{end}} +
+ Details + Visit +
+
+ + + + + + + + + {{range .Headers}} + {{if .IsInsecure}} + + {{else if .IsSecure}} + + {{else}} + + {{end}} + + + + {{end}} + +
HeaderValue
{{.Name}}{{.Value}}
+
+
+ {{end}} +
+ {{end}} + + + + + + + +` +) + +func (p *Page) AddHeader(name string, value string) { + p.Headers = append(p.Headers, Header{ + Name: name, + Value: value, + }) +} + +type ReportData struct { + Session *Session + Clusters [][]Page +} + +type Report struct { + Data ReportData +} + +func (r *Report) Render(dest io.Writer) error { + tmpl, err := template.New("Aquatone Report").Parse(Template) + if err != nil { + return err + } + err = tmpl.Execute(dest, r.Data) + if err != nil { + return err + } + return nil +} + +func NewReport(data ReportData) *Report { + return &Report{ + Data: data, + } +} + +func NewCluster(urls []string, session *Session) ([]Page, error) { + var cluster []Page + for _, url := range urls { + page, err := NewPage(url, session) + if err != nil { + continue + } + cluster = append(cluster, page) + } + + return cluster, nil +} + +func NewPage(url string, session *Session) (Page, error) { + baseFilename := session.BaseFilenameFromURL(url) + page := Page{ + URL: url, + HeadersPath: fmt.Sprintf("headers/%s.txt", baseFilename), + BodyPath: fmt.Sprintf("html/%s.html", baseFilename), + ScreenshotPath: fmt.Sprintf("screenshots/%s.png", baseFilename), + } + contents, err := session.ReadFile(fmt.Sprintf("headers/%s.txt", baseFilename)) + if err != nil { + return page, err + } + + if _, err := os.Stat(session.GetFilePath(page.ScreenshotPath)); os.IsNotExist(err) { + page.HasScreenshot = false + } else { + page.HasScreenshot = true + } + + lines := strings.Split(string(contents), "\n") + status, headers := lines[0], lines[1:] + page.Status = status + for _, header := range headers { + h := strings.Split(header, ": ") + if len(h) < 2 { + continue + } + name, value := h[0], strings.Join(h[1:], ": ") + page.AddHeader(name, value) + } + return page, nil +} diff --git a/core/session.go b/core/session.go new file mode 100644 index 0000000..000cac9 --- /dev/null +++ b/core/session.go @@ -0,0 +1,228 @@ +package core + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/asaskevich/EventBus" + "github.com/remeh/sizedwaitgroup" +) + +type Stats struct { + StartedAt time.Time + FinishedAt time.Time + PortOpen uint32 + PortClosed uint32 + RequestSuccessful uint32 + RequestFailed uint32 + ResponseCode2xx uint32 + ResponseCode3xx uint32 + ResponseCode4xx uint32 + ResponseCode5xx uint32 + ScreenshotSuccessful uint32 + ScreenshotFailed uint32 +} + +func (s *Stats) Duration() time.Duration { + return s.FinishedAt.Sub(s.StartedAt) +} + +func (s *Stats) IncrementPortOpen() { + atomic.AddUint32(&s.PortOpen, 1) +} + +func (s *Stats) IncrementPortClosed() { + atomic.AddUint32(&s.PortClosed, 1) +} + +func (s *Stats) IncrementRequestSuccessful() { + atomic.AddUint32(&s.RequestSuccessful, 1) +} + +func (s *Stats) IncrementRequestFailed() { + atomic.AddUint32(&s.RequestFailed, 1) +} + +func (s *Stats) IncrementResponseCode2xx() { + atomic.AddUint32(&s.ResponseCode2xx, 1) +} + +func (s *Stats) IncrementResponseCode3xx() { + atomic.AddUint32(&s.ResponseCode3xx, 1) +} + +func (s *Stats) IncrementResponseCode4xx() { + atomic.AddUint32(&s.ResponseCode4xx, 1) +} + +func (s *Stats) IncrementResponseCode5xx() { + atomic.AddUint32(&s.ResponseCode5xx, 1) +} + +func (s *Stats) IncrementScreenshotSuccessful() { + atomic.AddUint32(&s.ScreenshotSuccessful, 1) +} + +func (s *Stats) IncrementScreenshotFailed() { + atomic.AddUint32(&s.ScreenshotFailed, 1) +} + +type Session struct { + sync.Mutex + Version string + Options Options `json:"-"` + Out *Logger `json:"-"` + Stats *Stats + ResponsiveURLs []string + Ports []int + EventBus EventBus.Bus `json:"-"` + WaitGroup sizedwaitgroup.SizedWaitGroup `json:"-"` +} + +func (s *Session) Start() { + s.initStats() + s.initLogger() + s.initPorts() + s.initThreads() + s.initEventBus() + s.initWaitGroup() + s.initDirectories() +} + +func (s *Session) End() { + s.Stats.FinishedAt = time.Now() +} + +func (s *Session) initStats() { + if s.Stats != nil { + return + } + s.Stats = &Stats{ + StartedAt: time.Now(), + } +} + +func (s *Session) initPorts() { + var ports []int + switch *s.Options.Ports { + case "small": + ports = SmallPortList + case "", "medium", "default": + ports = MediumPortList + case "large": + ports = LargePortList + case "xlarge", "huge": + ports = XLargePortList + default: + for _, p := range strings.Split(*s.Options.Ports, ",") { + port, err := strconv.Atoi(strings.TrimSpace(p)) + if err != nil { + s.Out.Fatal("Invalid port range given\n") + os.Exit(1) + } + if port < 1 || port > 65535 { + s.Out.Fatal("Invalid port given: %v\n", port) + os.Exit(1) + } + ports = append(ports, port) + } + } + s.Ports = ports +} + +func (s *Session) initLogger() { + s.Out = &Logger{} + s.Out.SetDebug(*s.Options.Debug) + s.Out.SetSilent(*s.Options.Silent) +} + +func (s *Session) initThreads() { + if *s.Options.Threads == 0 { + numCPUs := runtime.NumCPU() + s.Options.Threads = &numCPUs + } +} + +func (s *Session) initEventBus() { + s.EventBus = EventBus.New() +} + +func (s *Session) initWaitGroup() { + s.WaitGroup = sizedwaitgroup.New(*s.Options.Threads) +} + +func (s *Session) initDirectories() { + for _, d := range []string{"headers", "html", "screenshots"} { + d = s.GetFilePath(d) + if _, err := os.Stat(d); os.IsNotExist(err) { + err = os.MkdirAll(d, 0755) + if err != nil { + s.Out.Fatal("Failed to create required directory %s\n", d) + os.Exit(1) + } + } + } +} + +func (s *Session) BaseFilenameFromURL(stru string) string { + u, err := url.Parse(stru) + if err != nil { + return "" + } + host := strings.Replace(u.Host, ":", "__", 1) + filename := fmt.Sprintf("%s__%s", u.Scheme, strings.Replace(host, ".", "_", -1)) + return strings.ToLower(filename) +} + +func (s *Session) GetFilePath(p string) string { + return path.Join(*s.Options.OutDir, p) +} + +func (s *Session) ReadFile(p string) ([]byte, error) { + content, err := ioutil.ReadFile(s.GetFilePath(p)) + if err != nil { + return content, err + } + return content, nil +} + +func (s *Session) AddResponsiveURL(url string) { + s.Lock() + defer s.Unlock() + s.ResponsiveURLs = append(s.ResponsiveURLs, url) +} + +func NewSession() (*Session, error) { + var err error + var session Session + + session.Version = Version + + if session.Options, err = ParseOptions(); err != nil { + return nil, err + } + + if *session.Options.ChromePath != "" { + if _, err := os.Stat(*session.Options.ChromePath); os.IsNotExist(err) { + return nil, fmt.Errorf("Chrome path %s does not exist", *session.Options.ChromePath) + } + } + + outdir := filepath.Clean(*session.Options.OutDir) + session.Options.OutDir = &outdir + + session.Version = Version + session.Start() + + return &session, nil +} diff --git a/core/similarity.go b/core/similarity.go new file mode 100644 index 0000000..eaceb81 --- /dev/null +++ b/core/similarity.go @@ -0,0 +1,28 @@ +package core + +import ( + "io" + + "github.com/pmezard/go-difflib/difflib" + "golang.org/x/net/html" +) + +func GetPageStructure(body io.Reader) ([]string, error) { + var structure []string + z := html.NewTokenizer(body) + for { + tt := z.Next() + switch tt { + case html.ErrorToken: + return structure, nil + case html.StartTagToken: + tn, _ := z.TagName() + structure = append(structure, string(tn)) + } + } +} + +func GetSimilarity(a, b []string) float64 { + matcher := difflib.NewMatcher(a, b) + return matcher.Ratio() +} diff --git a/core/urls.go b/core/urls.go new file mode 100644 index 0000000..08b6607 --- /dev/null +++ b/core/urls.go @@ -0,0 +1,43 @@ +package core + +import ( + "fmt" +) + +var ( + securePorts = []int{443, 832, 981, 1010, 1311, 2083, 2087, 2095, 2096, 4712, + 7000, 8172, 8243, 8333, 8443, 8834, 9443, 12443, 18091, 18092} +) + +func HostAndPortToURL(host string, port int, protocol string) string { + var url string + if protocol != "" { + url = fmt.Sprintf("%s://%s", protocol, host) + } else if isSecurePort(port) { + url = fmt.Sprintf("https://%s", host) + } else { + url = fmt.Sprintf("http://%s", host) + } + if isStandardPort(port) { + url = fmt.Sprintf("%s/", url) + } else { + url = fmt.Sprintf("%s:%d/", url, port) + } + return url +} + +func isSecurePort(port int) bool { + for _, p := range securePorts { + if p == port { + return true + } + } + return false +} + +func isStandardPort(port int) bool { + if port == 80 || port == 443 { + return true + } + return false +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..9f5f3ae --- /dev/null +++ b/main.go @@ -0,0 +1,201 @@ +package main + +import ( + "bufio" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/michenriksen/aquatone/agents" + "github.com/michenriksen/aquatone/core" + "github.com/michenriksen/aquatone/parsers" +) + +var ( + sess *core.Session + err error +) + +func isURL(s string) bool { + u, err := url.ParseRequestURI(s) + if err != nil { + return false + } + if u.Scheme == "" { + return false + } + return true +} + +func hasSupportedScheme(s string) bool { + u, err := url.ParseRequestURI(s) + if err != nil { + return false + } + if u.Scheme == "http" || u.Scheme == "https" { + return true + } + return false +} + +func main() { + if sess, err = core.NewSession(); err != nil { + fmt.Println(err) + os.Exit(1) + } + + fi, err := os.Stat(*sess.Options.OutDir) + + if os.IsNotExist(err) { + sess.Out.Fatal("Output destination %s does not exist\n", *sess.Options.OutDir) + os.Exit(1) + } + + if !fi.IsDir() { + sess.Out.Fatal("Output destination must be a directory\n") + os.Exit(1) + } + + sess.Out.Important("%s v%s started at %s\n\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) + + agents.NewTCPPortScanner().Register(sess) + agents.NewURLPublisher().Register(sess) + agents.NewURLRequester().Register(sess) + agents.NewURLLogger().Register(sess) + agents.NewURLScreenshotter().Register(sess) + + reader := bufio.NewReader(os.Stdin) + var targets []string + + if *sess.Options.Nmap { + parser := parsers.NewNmapParser() + targets, err = parser.Parse(reader) + if err != nil { + sess.Out.Fatal("Unable to parse input as Nmap/Masscan XML: %s\n", err) + os.Exit(1) + } + } else { + parser := parsers.NewRegexParser() + targets, err = parser.Parse(reader) + if err != nil { + sess.Out.Fatal("Unable to parse input.\n") + os.Exit(1) + } + } + + if len(targets) == 0 { + sess.Out.Fatal("No targets found in input.\n") + os.Exit(1) + } + + sess.Out.Important("Targets : %d\n", len(targets)) + sess.Out.Important("Threads : %d\n", *sess.Options.Threads) + sess.Out.Important("Ports : %s\n", strings.Trim(strings.Replace(fmt.Sprint(sess.Ports), " ", ", ", -1), "[]")) + sess.Out.Important("Output dir : %s\n\n", *sess.Options.OutDir) + + for _, target := range targets { + if isURL(target) { + if hasSupportedScheme(target) { + sess.EventBus.Publish(core.URL, target) + } + } else { + sess.EventBus.Publish(core.Host, target) + } + } + + sess.EventBus.WaitAsync() + sess.WaitGroup.Wait() + + sess.Out.Important("\nClustering similar sites...") + pageStructures := make(map[string][]string) + var pageClusters [][]string + + for _, responsiveURL := range sess.ResponsiveURLs { + filename := sess.GetFilePath(fmt.Sprintf("html/%s.html", agents.BaseFilenameFromURL(responsiveURL))) + body, err := os.Open(filename) + if err != nil { + continue + } + structure, _ := core.GetPageStructure(body) + pageStructures[responsiveURL] = structure + } + + // Loop over URL and page structure pairs + for url, structure := range pageStructures { + foundCluster := false + // Loop over existing page clusters + for i, cluster := range pageClusters { + addToCluster := true + // Loop over pages in cluster and check if similarity for all are 0.80 or above + for _, url2 := range cluster { + if core.GetSimilarity(structure, pageStructures[url2]) < 0.80 { + addToCluster = false + } + } + // Add to cluster if similarity between all pages are 0.80 or above + if addToCluster { + foundCluster = true + pageClusters[i] = append(pageClusters[i], url) + break + } + } + // If a cluster was not found for the page, create a new cluster for the page + if !foundCluster { + pageClusters = append(pageClusters, []string{url}) + } + } + + sess.Out.Important(" done\n") + sess.Out.Important("Generating HTML report...") + + reportData := core.ReportData{ + Session: sess, + } + + for _, urls := range pageClusters { + cluster, err := core.NewCluster(urls, sess) + if err != nil { + sess.Out.Fatal("Error during report generation: %s\n", err) + os.Exit(1) + } + reportData.Clusters = append(reportData.Clusters, cluster) + } + + report := core.NewReport(reportData) + f, err := os.OpenFile(sess.GetFilePath("aquatone_report.html"), os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + sess.Out.Fatal("Error during report generation: %s\n", err) + os.Exit(1) + } + err = report.Render(f) + if err != nil { + sess.Out.Fatal("Error during report generation: %s\n", err) + os.Exit(1) + } + + sess.Out.Important(" done\n\n") + + sess.End() + + sess.Out.Important("Time:\n") + sess.Out.Info(" - Started at : %v\n", sess.Stats.StartedAt.Format(time.RFC3339)) + sess.Out.Info(" - Finished at : %v\n", sess.Stats.FinishedAt.Format(time.RFC3339)) + sess.Out.Info(" - Duration : %v\n\n", sess.Stats.Duration().Round(time.Second)) + + sess.Out.Important("Requests:\n") + sess.Out.Info(" - Successful : %v\n", sess.Stats.RequestSuccessful) + sess.Out.Info(" - Failed : %v\n\n", sess.Stats.RequestFailed) + + sess.Out.Info(" - 2xx : %v\n", sess.Stats.ResponseCode2xx) + sess.Out.Info(" - 3xx : %v\n", sess.Stats.ResponseCode3xx) + sess.Out.Info(" - 4xx : %v\n", sess.Stats.ResponseCode4xx) + sess.Out.Info(" - 5xx : %v\n\n", sess.Stats.ResponseCode5xx) + + sess.Out.Important("Screenshots:\n") + sess.Out.Info(" - Successful : %v\n", sess.Stats.ScreenshotSuccessful) + sess.Out.Info(" - Failed : %v\n\n", sess.Stats.ScreenshotFailed) + + sess.Out.Important("Wrote HTML report to: %s\n\n", sess.GetFilePath("aquatone_report.html")) +} diff --git a/parsers/nmap.go b/parsers/nmap.go new file mode 100644 index 0000000..d38d06d --- /dev/null +++ b/parsers/nmap.go @@ -0,0 +1,79 @@ +package parsers + +import ( + "io" + "io/ioutil" + + "github.com/michenriksen/aquatone/core" + + "github.com/lair-framework/go-nmap" +) + +type NmapParser struct{} + +func NewNmapParser() *NmapParser { + return &NmapParser{} +} + +func (p *NmapParser) Parse(r io.Reader) ([]string, error) { + var targets []string + bytes, err := ioutil.ReadAll(r) + if err != nil { + return targets, err + } + scan, err := nmap.Parse(bytes) + if err != nil { + return targets, err + } + + for _, host := range scan.Hosts { + urls := p.hostToURLs(host) + for _, url := range urls { + targets = append(targets, url) + } + } + + return targets, nil +} + +func (p *NmapParser) isHTTPPort(port int) bool { + for _, p := range core.XLargePortList { + if p == port { + return true + } + } + return false +} + +func (p *NmapParser) hostToURLs(host nmap.Host) []string { + var urls []string + for _, port := range host.Ports { + var protocol string + if port.Service.Name == "ssl" { + protocol = "https" + } else if port.Service.Tunnel == "ssl" && (port.Service.Name != "smtp" && port.Service.Name != "imap" && port.Service.Name != "pop3") { + protocol = "https" + } else if port.Service.Name == "http" || port.Service.Name == "http-alt" { + protocol = "http" + } else { + if !p.isHTTPPort(port.PortId) { + continue + } + } + + if len(host.Hostnames) > 0 { + for _, hostname := range host.Hostnames { + urls = append(urls, core.HostAndPortToURL(hostname.Name, port.PortId, protocol)) + } + } else { + for _, address := range host.Addresses { + if address.AddrType == "mac" { + continue + } + urls = append(urls, core.HostAndPortToURL(address.Addr, port.PortId, protocol)) + } + } + } + + return urls +} diff --git a/parsers/regex.go b/parsers/regex.go new file mode 100644 index 0000000..9444fbe --- /dev/null +++ b/parsers/regex.go @@ -0,0 +1,31 @@ +package parsers + +import ( + "bufio" + "io" + + "github.com/mvdan/xurls" +) + +type RegexParser struct{} + +func NewRegexParser() *RegexParser { + return &RegexParser{} +} + +func (p *RegexParser) Parse(r io.Reader) ([]string, error) { + var targets []string + targetsFilter := make(map[string]struct{}) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + for _, target := range xurls.Relaxed().FindAllString(scanner.Text(), -1) { + if _, found := targetsFilter[target]; found { + continue + } + targets = append(targets, target) + targetsFilter[target] = struct{}{} + } + } + return targets, nil +} diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..b42e906 --- /dev/null +++ b/release.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +CURRENT_VERSION=$(cat core/banner.go | grep Version | cut -d '"' -f 2) +TO_UPDATE=( + core/banner.go +) + +read -p "[?] Did you remember to update CHANGELOG.md? " +read -p "[?] Did you remember to update README.md with new features/changes? " + +echo -n "[*] Current version is $CURRENT_VERSION. Enter new version: " +read NEW_VERSION +echo "[*] Pushing and tagging version $NEW_VERSION in 5 seconds..." +sleep 5 + +for file in "${TO_UPDATE[@]}" +do + echo "[*] Patching $file ..." + sed -i".bak" "s/$CURRENT_VERSION/$NEW_VERSION/g" $file + rm core/banner.go.bak + git add $file +done + +git commit -m "Releasing v$NEW_VERSION" +git push + +git tag -a v$NEW_VERSION -m "Release v$NEW_VERSION" +git push origin v$NEW_VERSION + +echo +echo "[*] All done, v$NEW_VERSION released."