diff --git a/go.mod b/go.mod index 109cd08..5cb75b3 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-isatty v0.0.13 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect - github.com/spyse-com/go-spyse v1.2.1 + github.com/spyse-com/go-spyse v1.2.4 github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect diff --git a/go.sum b/go.sum index 054ccf5..8ea0da1 100644 --- a/go.sum +++ b/go.sum @@ -34,7 +34,6 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -46,6 +45,10 @@ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxT github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8= github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw= +github.com/spyse-com/go-spyse v1.2.3 h1:0qo0OP5kLv0equyvI7H5pAGuDFTiFON3zXLv4BSw1yY= +github.com/spyse-com/go-spyse v1.2.3/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po= +github.com/spyse-com/go-spyse v1.2.4 h1:QEI+CHM2w4kCIkTihk0TQSIUupWueRhyBWf6L6LcYQY= +github.com/spyse-com/go-spyse v1.2.4/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -76,6 +79,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/input/flags.go b/input/flags.go index 4b473ca..b9e7d08 100644 --- a/input/flags.go +++ b/input/flags.go @@ -247,7 +247,7 @@ func ReadArgs() Input { examplesCommand.Parse(os.Args[2:]) default: output.Intro() - flag.PrintDefaults() + output.Help() os.Exit(1) } diff --git a/main.go b/main.go index 199b455..9cef2c2 100644 --- a/main.go +++ b/main.go @@ -50,31 +50,41 @@ func main() { func execute(userInput input.Input, subs map[string]output.Asset, dirs map[string]output.Asset, common []int) { var mutex = &sync.Mutex{} + var commandProvided = false // :::::::: REPORT SUBCOMMAND HANDLER :::::::: if userInput.ReportTarget != "" { + commandProvided = true ReportSubcommandHandler(userInput, mutex, dirs, subs) } // :::::::: DNS SUBCOMMAND HANDLER :::::::: if userInput.DNSTarget != "" { + commandProvided = true DNSSubcommandHandler(userInput) } // :::::::: SUBDOMAIN SUBCOMMAND HANDLER :::::::: if userInput.SubdomainTarget != "" { + commandProvided = true SubdomainSubcommandHandler(userInput, mutex, dirs, subs) } // :::::::: DIRECTORIES SUBCOMMAND HANDLER :::::::: if userInput.DirTarget != "" { + commandProvided = true DirSubcommandHandler(userInput, mutex, dirs, subs) } // :::::::: PORT SUBCOMMAND HANDLER :::::::: if userInput.PortTarget != "" { + commandProvided = true PortSubcommandHandler(userInput, common) } + + if !commandProvided { + output.Help() + } } //ReportSubcommandHandler > diff --git a/opendb/spyse.go b/opendb/spyse.go index 0214a44..77f0e5b 100644 --- a/opendb/spyse.go +++ b/opendb/spyse.go @@ -32,6 +32,9 @@ import ( spyse "github.com/spyse-com/go-spyse/pkg" ) +const searchMethodResultsLimit = 10000 +const defaultScrollResultsLimit = 20000 + //SpyseSubdomains appends to the subdomains in the list //thr subdomains found with the Spyse service. func SpyseSubdomains(target string, accessToken string) []string { @@ -45,9 +48,10 @@ func SpyseSubdomains(target string, accessToken string) []string { svc := spyse.NewDomainService(client) - //Dot before the domain name is important because search fetch any domains that end with ".$target" + //Dot before the domain name is important because search fetch any domains ending with ".$target" var searchDomain = "." + target var subdomainsSearchParams spyse.QueryBuilder + var ctx = context.Background() subdomainsSearchParams.AppendParam(spyse.QueryParam{ Name: svc.Params().Name.Name, @@ -55,24 +59,66 @@ func SpyseSubdomains(target string, accessToken string) []string { Value: searchDomain, }) - countResults, err := svc.SearchCount(context.Background(), subdomainsSearchParams.Query) + totalResults, err := svc.SearchCount(ctx, subdomainsSearchParams.Query) if err != nil { log.Fatal(err.Error()) } - var limit = 100 - var offset = 0 - var searchResults []spyse.Domain - var domain spyse.Domain - for ; int64(offset) < countResults; offset += limit { - //Notice that you can fetch only the first 10000 (can depend on your subscription plan) results using the Search method - searchResults, err = svc.Search(context.Background(), subdomainsSearchParams.Query, limit, offset) - if err != nil { - log.Fatal(err.Error()) + if totalResults == 0 { + return result + } + + // The default "Search" method returns only first 10 000 subdomains. + // To obtain more than 10 000 subdomains the "Scroll" method should be used. + // Note: The "Scroll" method is only available for "PRO" customers, so we need to check + // quota.IsScrollSearchEnabled parameter. + if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled { + var scrollID string + var scrollResults *spyse.DomainScrollResponse + + for { + if scrollResults, err = svc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID); err != nil { + if len(result) > 0 { + spyseErr, ok := err.(*spyse.ErrResponse) + if ok && spyseErr.Err.Code == spyse.CodeRequestsLimitReached { + break + } + } + log.Fatal(err.Error()) + } + if len(scrollResults.Items) > 0 { + scrollID = scrollResults.SearchID + + for _, domain := range scrollResults.Items { + result = append(result, domain.Name) + } + // The default "Scroll" limit, to avoid results that more than 20k + // If a limit of the number of requested subdomains will be added this block should be changed + if len(result) > defaultScrollResultsLimit { + break + } + } } - for _, domain = range searchResults { - result = append(result, domain.Name) + } else { + var limit = 100 + var searchResults []spyse.Domain + + for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { + if searchResults, err = svc.Search(ctx, subdomainsSearchParams.Query, limit, offset); err != nil { + if len(result) > 0 { + spyseErr, ok := err.(*spyse.ErrResponse) + if ok && spyseErr.Err.Code == spyse.CodeRequestsLimitReached { + break + } + } + log.Fatal(err.Error()) + } + + for _, domain := range searchResults { + result = append(result, domain.Name) + } } } + return result } diff --git a/output/intro.go b/output/intro.go index df639da..625c3d5 100644 --- a/output/intro.go +++ b/output/intro.go @@ -37,7 +37,7 @@ func Intro() { banner2 := " ___ ___(_) | | __ _\n" banner3 := " / __|/ __| | | |/ _` |\n" banner4 := " \\__ \\ (__| | | | (_| |\n" - banner5 := " |___/\\___|_|_|_|\\__,_| v1.1.1\n" + banner5 := " |___/\\___|_|_|_|\\__,_| v1.1.2\n" banner6 := " > github.com/edoardottt/scilla\n" banner7 := " > edoardoottavianelli.it" bannerPart1 := banner1 + banner2 + banner3 + banner4 + banner5 diff --git a/utils/url.go b/utils/url.go index d1353f2..264a438 100644 --- a/utils/url.go +++ b/utils/url.go @@ -78,7 +78,8 @@ func AppendDir(scheme string, domain string, dir string) (string, string) { return scheme + "://" + domain + "/" + dir + "/", scheme + "://" + domain + "/" + dir } -//CleanSubdomainsOk > +//CleanSubdomainsOk takes as input a slice of subdomains and remove +//from the input slice all the 'wrong' subdomains. func CleanSubdomainsOk(target string, input []string) []string { var result []string for _, elem := range input { @@ -87,7 +88,7 @@ func CleanSubdomainsOk(target string, input []string) []string { splits := strings.Split(elem, "\n") elem = splits[1] } - result = append(result, elem) + result = append(result, CleanProtocol(elem)) } } return result