From d8196979877e0db829dd855d69c7594fa0e04b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 11 Jul 2024 13:31:11 +0300 Subject: [PATCH 01/21] rename Jarm to JarmHash to avoid confusion --- runner/runner.go | 2 +- runner/types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 98cc1961..4661c9e4 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2204,7 +2204,7 @@ retry: FaviconURL: faviconURL, Hashes: hashesMap, Extracts: extractResult, - Jarm: jarmhash, + JarmHash: jarmhash, Lines: resp.Lines, Words: resp.Words, ASN: asnResponse, diff --git a/runner/types.go b/runner/types.go index 9c986ae1..da3b1cc6 100644 --- a/runner/types.go +++ b/runner/types.go @@ -67,7 +67,7 @@ type Result struct { RawHeaders string `json:"raw_header,omitempty" csv:"raw_header"` Request string `json:"request,omitempty" csv:"request"` ResponseTime string `json:"time,omitempty" csv:"time"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` + JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash"` ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` A []string `json:"a,omitempty" csv:"a"` AAAA []string `json:"aaaa,omitempty" csv:"aaaa"` From 6d3ef37158154d86d60557a9871f428dfa687939 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Fri, 12 Jul 2024 13:19:30 +0530 Subject: [PATCH 02/21] update fastdailer dep (#1818) * update fastdailer dep * use fastdailer main as dep --- go.mod | 12 ++++++------ go.sum | 26 ++++++++++++++------------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2a0ac40c..af07d6ae 100644 --- a/go.mod +++ b/go.mod @@ -24,20 +24,20 @@ require ( github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/dsl v0.1.2 - github.com/projectdiscovery/fastdialer v0.1.5 + github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.56 - github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/hmap v0.0.48 + github.com/projectdiscovery/gologger v1.1.13 + github.com/projectdiscovery/hmap v0.0.49 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/rawhttp v0.1.55 - github.com/projectdiscovery/retryablehttp-go v1.0.65 + github.com/projectdiscovery/retryablehttp-go v1.0.66 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.57 - github.com/projectdiscovery/utils v0.1.5 + github.com/projectdiscovery/utils v0.2.0 github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 @@ -114,7 +114,7 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.65 // indirect + github.com/projectdiscovery/retryabledns v1.0.66 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index 3e6743b0..8f71580d 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,10 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw= github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA= -github.com/projectdiscovery/fastdialer v0.1.5 h1:5rNJ8I+GVzDsUqRSVLUqGTm7LsshfNYmCh20lLn1aaI= -github.com/projectdiscovery/fastdialer v0.1.5/go.mod h1:oVoMl9Y6da2timhszy0okyBvhbKDSk+zqFazGV4y6pU= +github.com/projectdiscovery/fastdialer v0.1.7-0.20240710212120-b03f0ba5f1c9 h1:XFSUs5HfmtiRcrfBip8coC0DADxI0g8ag/KtH2opCRs= +github.com/projectdiscovery/fastdialer v0.1.7-0.20240710212120-b03f0ba5f1c9/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= +github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 h1:NvGOlJMEtBWkjb5r36VyoDzanG5IJzG2fDyszPavrck= +github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -243,12 +245,12 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= -github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= -github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E= +github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.48 h1:pbxoDcwZWSdaqZ3QgaG5Aw1G1qEFN2zeQco6eWeROAo= -github.com/projectdiscovery/hmap v0.0.48/go.mod h1:KMSbPJoIgI+9PkG6alZ0INEMRiIGnND128RUjxiY+8I= +github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc= +github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= @@ -259,18 +261,18 @@ github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSX github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/rawhttp v0.1.55 h1:OJQmiGRzmZqYNT58rsuPrSqVeNl9P3KjXmFor0Adaog= github.com/projectdiscovery/rawhttp v0.1.55/go.mod h1:+EAlybGLM2oh9ebdq+P9J0h7ma50jz/rrfzxiqN1t4c= -github.com/projectdiscovery/retryabledns v1.0.65 h1:wOtMuVkrcrGkzaT8Jixb3BtUcJODXLRz+z9EjftmFsQ= -github.com/projectdiscovery/retryabledns v1.0.65/go.mod h1:lf3xRdFhGh55fWVlzK1oAaDPrsKliqH3gGvsFNAOkCk= -github.com/projectdiscovery/retryablehttp-go v1.0.65 h1:GCZI9CUbYkldy9Iup4C7w6aVTxZuAHpSZTVy92qHPo4= -github.com/projectdiscovery/retryablehttp-go v1.0.65/go.mod h1:qF0UuglJiYJcezEoMIyYVYhHGwaOBy8bh3gRGz7CBs8= +github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY= +github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE= +github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU= +github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= -github.com/projectdiscovery/utils v0.1.5 h1:vjD2ZVtVEEiWNIKIFWkWSxaHnw8wGln8Lyfru6E+BW0= -github.com/projectdiscovery/utils v0.1.5/go.mod h1:NhjK2eVeoXLIZIYBJ2Z7yl6V4sB6Xr5Bzs1GdAqvJZ4= +github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= +github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 862e1a01d6e3ed016c8818f823893e9f066edc36 Mon Sep 17 00:00:00 2001 From: Ramana Reddy Date: Fri, 12 Jul 2024 22:42:46 +0530 Subject: [PATCH 03/21] fix test --- internal/testutils/integration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go index 2f5b3d09..b2bd35f3 100644 --- a/internal/testutils/integration.go +++ b/internal/testutils/integration.go @@ -26,7 +26,7 @@ func RunHttpxAndGetResults(url string, debug bool, extra ...string) ([]string, e return nil, err } parts := []string{} - items := strings.Split(string(data), "\n") + items := strings.Split(strings.TrimSpace(string(data)), "\n") for _, i := range items { if i != "" { parts = append(parts, i) @@ -72,7 +72,7 @@ func RunHttpxBinaryAndGetResults(target string, httpxBinary string, debug bool, return nil, err } parts := []string{} - items := strings.Split(string(data), "\n") + items := strings.Split(strings.TrimSpace(string(data)), "\n") for _, i := range items { if i != "" { parts = append(parts, i) From 276b7dae19a0f6cc06d69ea9a4c8c89c5c6c01af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:13:23 +0000 Subject: [PATCH 04/21] chore(deps): bump github.com/projectdiscovery/utils from 0.2.0 to 0.2.1 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.0 to 0.2.1. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.0...v0.2.1) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index af07d6ae..35fd44d3 100644 --- a/go.mod +++ b/go.mod @@ -28,16 +28,16 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.56 - github.com/projectdiscovery/gologger v1.1.13 + github.com/projectdiscovery/gologger v1.1.14 github.com/projectdiscovery/hmap v0.0.49 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/rawhttp v0.1.55 - github.com/projectdiscovery/retryablehttp-go v1.0.66 + github.com/projectdiscovery/retryablehttp-go v1.0.68 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.57 - github.com/projectdiscovery/utils v0.2.0 + github.com/projectdiscovery/utils v0.2.1 github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index 8f71580d..50515827 100644 --- a/go.sum +++ b/go.sum @@ -233,8 +233,6 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw= github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA= -github.com/projectdiscovery/fastdialer v0.1.7-0.20240710212120-b03f0ba5f1c9 h1:XFSUs5HfmtiRcrfBip8coC0DADxI0g8ag/KtH2opCRs= -github.com/projectdiscovery/fastdialer v0.1.7-0.20240710212120-b03f0ba5f1c9/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 h1:NvGOlJMEtBWkjb5r36VyoDzanG5IJzG2fDyszPavrck= github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= @@ -245,8 +243,8 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= -github.com/projectdiscovery/gologger v1.1.13 h1:7h4yCnMk+DQ/43v+A3NuA1uuI4XlJd0W0xvTZioD05E= -github.com/projectdiscovery/gologger v1.1.13/go.mod h1:PAsVPr2aHoopCzE0BOnp2bPCv+BU/Oq81ygcP3xVwgI= +github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= +github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc= @@ -263,16 +261,16 @@ github.com/projectdiscovery/rawhttp v0.1.55 h1:OJQmiGRzmZqYNT58rsuPrSqVeNl9P3KjX github.com/projectdiscovery/rawhttp v0.1.55/go.mod h1:+EAlybGLM2oh9ebdq+P9J0h7ma50jz/rrfzxiqN1t4c= github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY= github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE= -github.com/projectdiscovery/retryablehttp-go v1.0.66 h1:dY+sGVf/O8CvKcSdTx0rxJoKG+HyOScz5T89TijZohU= -github.com/projectdiscovery/retryablehttp-go v1.0.66/go.mod h1:3QJt06j4hEZVpJOr31u49I3mDnqbAzhcjJX8Yvn/xnY= +github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I= +github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= -github.com/projectdiscovery/utils v0.2.0 h1:GC9FOq2mVacVYZgD2s6XW7RP6jAC/XjOKclSpLYPUUQ= -github.com/projectdiscovery/utils v0.2.0/go.mod h1:kOnMoFfuQMY8U+nh8e32odjC00iEbiLVBcDxyAZFwCc= +github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk= +github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4= github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= From 95a39b90b26e688daba8f8381ad2b47bf6bcf19d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:19:05 +0000 Subject: [PATCH 05/21] chore(deps): bump github.com/projectdiscovery/useragent Bumps [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) from 0.0.57 to 0.0.59. - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.57...v0.0.59) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 35fd44d3..57542304 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.56 + github.com/projectdiscovery/goflags v0.1.58 github.com/projectdiscovery/gologger v1.1.14 github.com/projectdiscovery/hmap v0.0.49 github.com/projectdiscovery/mapcidr v1.1.34 @@ -36,7 +36,7 @@ require ( github.com/projectdiscovery/rawhttp v0.1.55 github.com/projectdiscovery/retryablehttp-go v1.0.68 github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/useragent v0.0.57 + github.com/projectdiscovery/useragent v0.0.59 github.com/projectdiscovery/utils v0.2.1 github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/rs/xid v1.5.0 diff --git a/go.sum b/go.sum index 50515827..dd246018 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.56 h1:tJYiZN7s9Jk9DxfYOUiqOoybaIDlXyX4ZgT4B/06SyU= -github.com/projectdiscovery/goflags v0.1.56/go.mod h1:DsGF0NPpM5hGg75N3MTSvWJ4MIT7HFEAOEeWZ074+Fg= +github.com/projectdiscovery/goflags v0.1.58 h1:+NRyb4/YcMFXPJvd2C+29V/Hv5Hx3vVkbgcEQybphGc= +github.com/projectdiscovery/goflags v0.1.58/go.mod h1:CFXs1uZYCuGU9fYdGxQ8CkjwssLDwPPGdQ5CEha+iZ8= github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -267,8 +267,8 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/useragent v0.0.57 h1:y6ML3GVJ6Sq+nhqk2gY9aD0JNaKzfpunJPjPN40eVN4= -github.com/projectdiscovery/useragent v0.0.57/go.mod h1:iChqL4rGApUsdXhAzItirEpMvYYOlruieZE+nqoKpCc= +github.com/projectdiscovery/useragent v0.0.59 h1:KBc3KP0EIDTh/0jk/xNph+KsMnQSQffsQnbHV+ex4Lo= +github.com/projectdiscovery/useragent v0.0.59/go.mod h1:Y5ye6dJL53ySnIhejleRwuewIv424sxaEVoxZnVm30w= github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk= github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4= github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= From fd4fbe0842dbb4e3f965c0f80b08b7178478ce52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:19:09 +0000 Subject: [PATCH 06/21] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.2 to 0.1.6 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.2 to 0.1.6. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.2...v0.1.6) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 35fd44d3..2f288fa4 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/projectdiscovery/asnmap v1.1.1 github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/dsl v0.1.2 + github.com/projectdiscovery/dsl v0.1.6 github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 diff --git a/go.sum b/go.sum index 50515827..f81b202f 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.2 h1:RdWCEsUqyWsLIQ1k16NX5FiIVbPFvzx9BOlvzK7eeGw= -github.com/projectdiscovery/dsl v0.1.2/go.mod h1:3zwogxGPnPkVUk5OfZQ9Y51sAWdazfQoVZFSPqeEUYA= +github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= +github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 h1:NvGOlJMEtBWkjb5r36VyoDzanG5IJzG2fDyszPavrck= github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= From 11e07351b5954d6f71fc6b2cecde6297d5fd18c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:19:16 +0000 Subject: [PATCH 07/21] chore(deps): bump github.com/projectdiscovery/retryabledns Bumps [github.com/projectdiscovery/retryabledns](https://github.com/projectdiscovery/retryabledns) from 1.0.66 to 1.0.67. - [Release notes](https://github.com/projectdiscovery/retryabledns/releases) - [Commits](https://github.com/projectdiscovery/retryabledns/compare/v1.0.66...v1.0.67) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryabledns dependency-type: indirect update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 35fd44d3..0634ec53 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,7 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.66 // indirect + github.com/projectdiscovery/retryabledns v1.0.67 // indirect github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect diff --git a/go.sum b/go.sum index 50515827..1c53102e 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSX github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/rawhttp v0.1.55 h1:OJQmiGRzmZqYNT58rsuPrSqVeNl9P3KjXmFor0Adaog= github.com/projectdiscovery/rawhttp v0.1.55/go.mod h1:+EAlybGLM2oh9ebdq+P9J0h7ma50jz/rrfzxiqN1t4c= -github.com/projectdiscovery/retryabledns v1.0.66 h1:OzKcR+TqIYdSDEED2JWJA90ZNFCEKRv+hOvpMHEZEOY= -github.com/projectdiscovery/retryabledns v1.0.66/go.mod h1:ODpvZ6kXXB2uqK7GPyfkd/tR+4wkeIm1DR5TkRQv7KE= +github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= +github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I= github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= From 576e25f4c6e5b76e4ad16a12291a21707f886f9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:49:15 +0000 Subject: [PATCH 08/21] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.14 to 1.1.16. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.14...v1.1.16) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bc770dcf..4018782a 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.58 - github.com/projectdiscovery/gologger v1.1.14 + github.com/projectdiscovery/gologger v1.1.16 github.com/projectdiscovery/hmap v0.0.49 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 diff --git a/go.sum b/go.sum index 82228a83..94c06c09 100644 --- a/go.sum +++ b/go.sum @@ -243,8 +243,8 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.58 h1:+NRyb4/YcMFXPJvd2C+29V/Hv5Hx3vVkbgcEQybphGc= github.com/projectdiscovery/goflags v0.1.58/go.mod h1:CFXs1uZYCuGU9fYdGxQ8CkjwssLDwPPGdQ5CEha+iZ8= -github.com/projectdiscovery/gologger v1.1.14 h1:8SS9qDCisCFffKHzWMX+GDDArxBJ9A7DhYrESEixAYo= -github.com/projectdiscovery/gologger v1.1.14/go.mod h1:CPk1nAZ3PqgspKBuSJR+xwY2i/Rm+P/ovahpP30EJy8= +github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc= +github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc= From 02731e962951540687a3aead661b8e5364f0f89b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:55:15 +0000 Subject: [PATCH 09/21] chore(deps): bump github.com/projectdiscovery/hmap from 0.0.49 to 0.0.51 Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.49 to 0.0.51. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.49...v0.0.51) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4018782a..1f7999e4 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.58 github.com/projectdiscovery/gologger v1.1.16 - github.com/projectdiscovery/hmap v0.0.49 + github.com/projectdiscovery/hmap v0.0.51 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 diff --git a/go.sum b/go.sum index 94c06c09..935473bf 100644 --- a/go.sum +++ b/go.sum @@ -247,8 +247,8 @@ github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bj github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.49 h1:QNW98JDqd0nmaIFRMbn0sLocRIGXAud1liGip4iPdvc= -github.com/projectdiscovery/hmap v0.0.49/go.mod h1:BA4cyYTrWKMUa78815R6p3tuGiYu2MJHYnjwXmefdbU= +github.com/projectdiscovery/hmap v0.0.51 h1:xqbpRAJRHPMoS2uERkbWGObIO4bv+whe3PEk3h4lDEg= +github.com/projectdiscovery/hmap v0.0.51/go.mod h1:vqdeWnNVMJYyIDytu+IdJDFg3wZdRVN83AKHR40RP6c= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= From 640be33c4411390684298bda4327a766d245af57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 11:01:08 +0000 Subject: [PATCH 10/21] chore(deps): bump github.com/projectdiscovery/utils from 0.2.1 to 0.2.2 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.1 to 0.2.2. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.1...v0.2.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 12 +++++------- go.sum | 32 ++++++++++---------------------- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 1f7999e4..11efa713 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/dsl v0.1.6 - github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 + github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.58 @@ -34,10 +34,10 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/rawhttp v0.1.55 - github.com/projectdiscovery/retryablehttp-go v1.0.68 + github.com/projectdiscovery/retryablehttp-go v1.0.69 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.59 - github.com/projectdiscovery/utils v0.2.1 + github.com/projectdiscovery/utils v0.2.2 github.com/projectdiscovery/wappalyzergo v0.1.8 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 @@ -78,7 +78,6 @@ require ( github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.9.5 // indirect - github.com/gaukas/godicttls v0.0.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect @@ -90,7 +89,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kataras/jwt v0.1.8 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/pgzip v1.2.5 // indirect github.com/kljensen/snowball v0.8.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -115,8 +114,7 @@ require ( github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/retryabledns v1.0.67 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect - github.com/refraction-networking/utls v1.5.4 // indirect + github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect diff --git a/go.sum b/go.sum index 935473bf..bf585950 100644 --- a/go.sum +++ b/go.sum @@ -72,18 +72,12 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= -github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= -github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI= github.com/go-faker/faker/v4 v4.1.1/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -113,8 +107,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= @@ -140,8 +132,8 @@ github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -209,8 +201,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= @@ -233,8 +223,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= -github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85 h1:NvGOlJMEtBWkjb5r36VyoDzanG5IJzG2fDyszPavrck= -github.com/projectdiscovery/fastdialer v0.1.7-0.20240711151717-033b71c47e85/go.mod h1:cM0bEbI9wAqkL2eJr5Ll95TSCcBCDYM9oy7eDFcFgkA= +github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= +github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -261,22 +251,20 @@ github.com/projectdiscovery/rawhttp v0.1.55 h1:OJQmiGRzmZqYNT58rsuPrSqVeNl9P3KjX github.com/projectdiscovery/rawhttp v0.1.55/go.mod h1:+EAlybGLM2oh9ebdq+P9J0h7ma50jz/rrfzxiqN1t4c= github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= -github.com/projectdiscovery/retryablehttp-go v1.0.68 h1:MN/kfccDnebmp1kNcCnGlGuCfvet6y5eN1Y1hIKBB1I= -github.com/projectdiscovery/retryablehttp-go v1.0.68/go.mod h1:F3QNCoJExPFT59AACrVgYVhNiDW5Fp8s5HbXc+8CaaU= +github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= +github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/useragent v0.0.59 h1:KBc3KP0EIDTh/0jk/xNph+KsMnQSQffsQnbHV+ex4Lo= github.com/projectdiscovery/useragent v0.0.59/go.mod h1:Y5ye6dJL53ySnIhejleRwuewIv424sxaEVoxZnVm30w= -github.com/projectdiscovery/utils v0.2.1 h1:XcOUJtR5ReXDWqxDkJWPVpC2lSqIYTs8z8YNILcPiqk= -github.com/projectdiscovery/utils v0.2.1/go.mod h1:h3o2zmJguUm9FKvel1F9jYCLxoWhFakyytRMjqb7Dg4= +github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= +github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= -github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= -github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= +github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= +github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= From 829fa4e18e66b96d51852b61f6a718964cad1913 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 19:49:18 +0000 Subject: [PATCH 11/21] chore(deps): bump github.com/projectdiscovery/dsl from 0.1.6 to 0.1.7 Bumps [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) from 0.1.6 to 0.1.7. - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.1.6...v0.1.7) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 11efa713..e3565279 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/projectdiscovery/asnmap v1.1.1 github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/dsl v0.1.6 + github.com/projectdiscovery/dsl v0.1.7 github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 diff --git a/go.sum b/go.sum index bf585950..bce0164f 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.6 h1:6TUYMwbjcYJ5OoRoawPX1SYD7wgubX/1FreC8SwFqgc= -github.com/projectdiscovery/dsl v0.1.6/go.mod h1:1PAV9A6X+fdyMXW0SY4i0BHrO8yN2+FB2jp25+0AT04= +github.com/projectdiscovery/dsl v0.1.7 h1:mPaHFPr2IMq69SjFGTt/DP8W5QeaGo+6lHqusnko2Qw= +github.com/projectdiscovery/dsl v0.1.7/go.mod h1:pjBGO539itAgO+qD5PSXTcvRiF92tJKLzkA1aqCMwEQ= github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= From def4e7767b7155f27c61fb95363e154e6497f5ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 19:49:59 +0000 Subject: [PATCH 12/21] chore(deps): bump github.com/projectdiscovery/goflags Bumps [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) from 0.1.58 to 0.1.60. - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.58...v0.1.60) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 11efa713..25436e91 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/projectdiscovery/fastdialer v0.2.0 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.58 + github.com/projectdiscovery/goflags v0.1.60 github.com/projectdiscovery/gologger v1.1.16 github.com/projectdiscovery/hmap v0.0.51 github.com/projectdiscovery/mapcidr v1.1.34 diff --git a/go.sum b/go.sum index bf585950..fcb1d070 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvm github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.58 h1:+NRyb4/YcMFXPJvd2C+29V/Hv5Hx3vVkbgcEQybphGc= -github.com/projectdiscovery/goflags v0.1.58/go.mod h1:CFXs1uZYCuGU9fYdGxQ8CkjwssLDwPPGdQ5CEha+iZ8= +github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= +github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc= github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= From 63b64cb4e6063f11511d849cc58a9f598bcd474a Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Date: Fri, 26 Jul 2024 20:57:19 +0530 Subject: [PATCH 13/21] feat: httpx results dashboard (#1834) * misc updates * WIP: add dashboard support * feat dashboard support * screenshot with upload validation * misc textual update * interactive api key login * feat: add -pdu local results file upload * fix missing tech info in dashboard * update flag description --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- cmd/httpx/httpx.go | 83 ++++++++++++- go.mod | 60 ++++----- go.sum | 148 +++++++++++----------- internal/pdcp/utils.go | 19 +++ internal/pdcp/writer.go | 263 ++++++++++++++++++++++++++++++++++++++++ runner/banner.go | 6 +- runner/healthcheck.go | 2 +- runner/options.go | 26 +++- runner/runner.go | 36 +++--- 9 files changed, 515 insertions(+), 128 deletions(-) create mode 100644 internal/pdcp/utils.go create mode 100644 internal/pdcp/writer.go diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 1a233e9f..3c890e29 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -1,14 +1,19 @@ package main import ( + "context" + "encoding/json" "os" "os/signal" "runtime" "runtime/pprof" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/internal/pdcp" "github.com/projectdiscovery/httpx/runner" - errorutil "github.com/projectdiscovery/utils/errors" + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" + _ "github.com/projectdiscovery/utils/pprof" ) func main() { @@ -33,6 +38,30 @@ func main() { }() } + // validation for local results file upload + if options.AssetFileUpload != "" { + _ = setupOptionalAssetUpload(options) + file, err := os.Open(options.AssetFileUpload) + if err != nil { + gologger.Fatal().Msgf("Could not open file: %s\n", err) + } + defer file.Close() + dec := json.NewDecoder(file) + for dec.More() { + var r runner.Result + err := dec.Decode(&r) + if err != nil { + gologger.Fatal().Msgf("Could not decode jsonl file: %s\n", err) + } + options.OnResult(r) + } + options.OnClose() + return + } + + // setup optional asset upload + _ = setupOptionalAssetUpload(options) + httpxRunner, err := runner.New(options) if err != nil { gologger.Fatal().Msgf("Could not create runner: %s\n", err) @@ -60,8 +89,54 @@ func main() { httpxRunner.Close() } -func init() { - if os.Getenv("DEBUG") != "" { - errorutil.ShowStackTrace = true +// setupOptionalAssetUpload is used to setup optional asset upload +// this is optional and only initialized when explicitly enabled +func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter { + var mustEnable bool + // enable on multiple conditions + if opts.AssetUpload || opts.AssetID != "" || opts.AssetName != "" || pdcp.EnableCloudUpload { + mustEnable = true + } + a := aurora.NewAurora(!opts.NoColor) + if !mustEnable { + if !pdcp.HideAutoSaveMsg { + gologger.Print().Msgf("[%s] UI Dashboard is disabled, Use -dashboard option to enable", a.BrightYellow("WRN")) + } + return nil + } + if opts.Screenshot { + gologger.Fatal().Msgf("Screenshot option is not supported for dashboard upload yet") + } + gologger.Info().Msgf("To view results in UI dashboard, visit https://cloud.projectdiscovery.io/assets upon completion.") + h := &pdcpauth.PDCPCredHandler{} + creds, err := h.GetCreds() + if err != nil { + if err != pdcpauth.ErrNoCreds && !pdcp.HideAutoSaveMsg { + gologger.Verbose().Msgf("Could not get credentials for cloud upload: %s\n", err) + } + pdcpauth.CheckNValidateCredentials("httpx") + return nil + } + writer, err := pdcp.NewUploadWriterCallback(context.Background(), creds) + if err != nil { + gologger.Error().Msgf("failed to setup UI dashboard: %s", err) + return nil + } + if writer == nil { + gologger.Error().Msgf("something went wrong, could not setup UI dashboard") + } + opts.OnResult = writer.GetWriterCallback() + opts.OnClose = func() { + writer.Close() + } + // add additional metadata + if opts.AssetID != "" { + // silently ignore + _ = writer.SetAssetID(opts.AssetID) + } + if opts.AssetName != "" { + // silently ignore + writer.SetAssetGroupName(opts.AssetName) } + return writer } diff --git a/go.mod b/go.mod index d4ef05e3..23781101 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 github.com/PuerkitoBio/goquery v1.8.1 - github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/akrylysov/pogreb v0.10.2 // indirect github.com/corona10/goimagehash v1.1.0 github.com/go-faker/faker/v4 v4.1.1 github.com/go-rod/rod v0.114.0 @@ -17,7 +17,7 @@ require ( github.com/logrusorgru/aurora v2.0.3+incompatible github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/microcosm-cc/bluemonday v1.0.26 - github.com/miekg/dns v1.1.56 // indirect + github.com/miekg/dns v1.1.59 // indirect github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 @@ -33,25 +33,25 @@ require ( github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 - github.com/projectdiscovery/rawhttp v0.1.55 + github.com/projectdiscovery/rawhttp v0.1.57 github.com/projectdiscovery/retryablehttp-go v1.0.69 github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/useragent v0.0.59 + github.com/projectdiscovery/useragent v0.0.60 github.com/projectdiscovery/utils v0.2.2 - github.com/projectdiscovery/wappalyzergo v0.1.8 + github.com/projectdiscovery/wappalyzergo v0.1.10 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.9.0 - github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 - go.etcd.io/bbolt v1.3.7 // indirect + github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db + go.etcd.io/bbolt v1.3.10 // indirect go.uber.org/multierr v1.11.0 - golang.org/x/exp v0.0.0-20230810033253-352e893a4cad + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.26.0 golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 ) -require github.com/weppos/publicsuffix-go v0.30.1 +require github.com/weppos/publicsuffix-go v0.30.2 require ( aead.dev/minisign v0.2.0 // indirect @@ -60,7 +60,7 @@ require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect - github.com/andybalholm/brotli v1.0.6 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect @@ -69,34 +69,34 @@ require ( github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/cfssl v1.6.4 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.3.8 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.11.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect - github.com/fatih/color v1.15.0 // indirect + github.com/fatih/color v1.16.0 // indirect github.com/gaissmai/bart v0.9.5 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/certificate-transparency-go v1.1.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/gorilla/css v1.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kataras/jwt v0.1.8 // indirect - github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/kataras/jwt v0.1.10 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kljensen/snowball v0.8.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -106,7 +106,7 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pierrec/lz4/v4 v4.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect @@ -118,14 +118,14 @@ require ( github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect - github.com/sashabaranov/go-openai v1.14.2 // indirect - github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/sashabaranov/go-openai v1.15.3 // indirect + github.com/shirou/gopsutil/v3 v3.24.2 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/tidwall/btree v1.6.0 // indirect - github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/btree v1.7.0 // indirect + github.com/tidwall/buntdb v1.3.1 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -133,7 +133,7 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/ysmood/fetchup v0.2.3 // indirect github.com/ysmood/goob v0.4.0 // indirect @@ -147,12 +147,12 @@ require ( github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 4a496201..4ba831ad 100644 --- a/go.sum +++ b/go.sum @@ -15,13 +15,13 @@ github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJs github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= -github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78= +github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= -github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= @@ -44,8 +44,8 @@ github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4 github.com/cloudflare/cfssl v1.6.4 h1:NMOvfrEjFfC63K3SGXgAnFdsgkmiq4kATme5BfcqrO8= github.com/cloudflare/cfssl v1.6.4/go.mod h1:8b3CQMxfWPAeom3zBnGJ6sd+G1NkL5TXqmDXacb+1J0= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI= +github.com/cloudflare/circl v1.3.8/go.mod h1:PDRU+oXvdD7KCtgKxW95M5Z8BpSCJXQORiZFnBQS5QU= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corona10/goimagehash v1.1.0 h1:teNMX/1e+Wn/AYSbLHX8mj+mF9r60R1kBeqE9MkoYwI= @@ -56,15 +56,15 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -84,8 +84,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -101,7 +101,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= -github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-github/v50 v50.2.0/go.mod h1:VBY8FB6yPIjrtKhozXv4FQupxKLS6H4m6xFZlT43q8Q= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -109,10 +108,11 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= @@ -128,15 +128,16 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= -github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= +github.com/kataras/jwt v0.1.10 h1:GBXOF9RVInDPhCFBiDumRG9Tt27l7ugLeLo8HL5SeKQ= +github.com/kataras/jwt v0.1.10/go.mod h1:xkimAtDhU/aGlQqjwvgtg+VyuPwMiyZHaY8LJRh0mYo= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kljensen/snowball v0.8.0 h1:WU4cExxK6sNW33AiGdbn4e8RvloHrhkAssu2mVJ11kg= github.com/kljensen/snowball v0.8.0/go.mod h1:OGo5gFWjaeXqCu4iIrMl5OYip9XUJHGOU5eSkPjVg2A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -156,12 +157,13 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= @@ -170,8 +172,8 @@ github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXm github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= -github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -205,8 +207,9 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -247,8 +250,8 @@ github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+ github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= -github.com/projectdiscovery/rawhttp v0.1.55 h1:OJQmiGRzmZqYNT58rsuPrSqVeNl9P3KjXmFor0Adaog= -github.com/projectdiscovery/rawhttp v0.1.55/go.mod h1:+EAlybGLM2oh9ebdq+P9J0h7ma50jz/rrfzxiqN1t4c= +github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko= +github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE= github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= @@ -257,12 +260,12 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/useragent v0.0.59 h1:KBc3KP0EIDTh/0jk/xNph+KsMnQSQffsQnbHV+ex4Lo= -github.com/projectdiscovery/useragent v0.0.59/go.mod h1:Y5ye6dJL53ySnIhejleRwuewIv424sxaEVoxZnVm30w= +github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc= +github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w= github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= -github.com/projectdiscovery/wappalyzergo v0.1.8 h1:10cwDuxO6TC6uGbIRCrVcXCjTUH0zbrX7WQIxg/pPDI= -github.com/projectdiscovery/wappalyzergo v0.1.8/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= +github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -275,17 +278,17 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/sashabaranov/go-openai v1.14.2 h1:5DPTtR9JBjKPJS008/A409I5ntFhUPPGCmaAihcPRyo= -github.com/sashabaranov/go-openai v1.14.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= -github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= -github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/sashabaranov/go-openai v1.15.3 h1:rzoNK9n+Cak+PM6OQ9puxDmFllxfnVea9StlmhglXqA= +github.com/sashabaranov/go-openai v1.15.3/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/shirou/gopsutil/v3 v3.24.2 h1:kcR0erMbLg5/3LcInpw0X/rrPSqq4CDPyI6A6ZRC18Y= +github.com/shirou/gopsutil/v3 v3.24.2/go.mod h1:tSg/594BcA+8UdQU2XcW803GWYgdtauFFPgJCJKZlVk= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= @@ -307,13 +310,13 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= -github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= -github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= -github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= -github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= +github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o= +github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -327,20 +330,17 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= -github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= -github.com/weppos/publicsuffix-go v0.30.1/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= +github.com/weppos/publicsuffix-go v0.30.2 h1:Np18yzfMR90jNampWFs7iSh2sw/qCZkhL41/ffyihCU= +github.com/weppos/publicsuffix-go v0.30.2/go.mod h1:/hGscit36Yt+wammfBBwdMdxBT8btsTt6KvwO9OvMyM= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= @@ -366,7 +366,6 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= @@ -378,11 +377,11 @@ github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54t github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= -github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db h1:IfONOhyZlf4qPt3ENPU+27mBbPjzTQ+swKpj7MJva9I= +github.com/zmap/zcrypto v0.0.0-20240512203510-0fef58d9a9db/go.mod h1:mo/07mo6reDaiz6BzveCuYBWb1d+aX8Pf8Nh+Q57y2g= github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -393,16 +392,16 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -427,14 +426,15 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -464,13 +464,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -480,7 +481,9 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -489,10 +492,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -508,12 +514,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/pdcp/utils.go b/internal/pdcp/utils.go new file mode 100644 index 00000000..301ef2aa --- /dev/null +++ b/internal/pdcp/utils.go @@ -0,0 +1,19 @@ +package pdcp + +import ( + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" + urlutil "github.com/projectdiscovery/utils/url" +) + +func getAssetsDashBoardURL(id string) string { + ux, _ := urlutil.Parse(pdcpauth.DashBoardURL) + ux.Path = "/assets/" + id + ux.Update() + return ux.String() +} + +// {"asset_id":"cqdtekhte9oc73e9hrvg","message":"Successfully uploaded asset","upload_status":"success","uploaded_at":"2024-07-20 15:27:16.148527329 +0000 UTC m=+1078.215945902"} +type uploadResponse struct { + ID string `json:"asset_id"` + Message string `json:"message"` +} diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go new file mode 100644 index 00000000..5fd023ab --- /dev/null +++ b/internal/pdcp/writer.go @@ -0,0 +1,263 @@ +package pdcp + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "sync/atomic" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/runner" + "github.com/projectdiscovery/retryablehttp-go" + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" + "github.com/projectdiscovery/utils/conversion" + "github.com/projectdiscovery/utils/env" + errorutil "github.com/projectdiscovery/utils/errors" + unitutils "github.com/projectdiscovery/utils/unit" + updateutils "github.com/projectdiscovery/utils/update" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + uploadEndpoint = "/v1/assets" + appendEndpoint = "/v1/assets/%s/contents" + flushTimer = time.Minute + MaxChunkSize = 4 * unitutils.Mega // 4 MB + xidRe = `^[a-z0-9]{20}$` + teamIDHeader = "X-Team-Id" +) + +var ( + xidRegex = regexp.MustCompile(xidRe) + // teamID if given + teamID = env.GetEnvOrDefault("PDCP_TEAM_ID", "") + // EnableeUpload if set to true enables the upload feature + HideAutoSaveMsg = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD_WRN", false) + EnableCloudUpload = env.GetEnvOrDefault("ENABLE_CLOUD_UPLOAD", false) +) + +// UploadWriter is a writer that uploads its output to pdcp +// server to enable web dashboard and more +type UploadWriter struct { + creds *pdcpauth.PDCPCredentials + uploadURL *url.URL + client *retryablehttp.Client + done chan struct{} + data chan runner.Result + assetGroupID string + assetGroupName string + counter atomic.Int32 + closed atomic.Bool +} + +// NewUploadWriterCallback creates a new upload writer callback +// which when enabled periodically uploads the results to pdcp assets dashboard +func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredentials) (*UploadWriter, error) { + if creds == nil { + return nil, fmt.Errorf("no credentials provided") + } + u := &UploadWriter{ + creds: creds, + done: make(chan struct{}, 1), + data: make(chan runner.Result, 8), // default buffer size + } + var err error + tmp, err := urlutil.Parse(creds.Server) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not parse server url") + } + tmp.Path = uploadEndpoint + tmp.Update() + u.uploadURL = tmp.URL + + // create http client + opts := retryablehttp.DefaultOptionsSingle + opts.NoAdjustTimeout = true + opts.Timeout = time.Duration(3) * time.Minute + u.client = retryablehttp.NewClient(opts) + // start auto commit + // upload every 1 minute or when buffer is full + go u.autoCommit(ctx) + return u, nil +} + +// GetWriterCallback returns the writer callback +func (u *UploadWriter) GetWriterCallback() runner.OnResultCallback { + return func(r runner.Result) { + if r.Err != nil { + return + } + u.data <- r + } +} + +// SetAssetID sets the scan id for the upload writer +func (u *UploadWriter) SetAssetID(id string) error { + if !xidRegex.MatchString(id) { + return fmt.Errorf("invalid asset id provided") + } + u.assetGroupID = id + return nil +} + +// SetAssetGroupName sets the scan name for the upload writer +func (u *UploadWriter) SetAssetGroupName(name string) { + u.assetGroupName = name +} + +func (u *UploadWriter) autoCommit(ctx context.Context) { + // wait for context to be done + defer func() { + u.done <- struct{}{} + close(u.done) + // if no scanid is generated no results were uploaded + if u.assetGroupID == "" { + gologger.Verbose().Msgf("UI dashboard setup skipped, no results found to upload") + } else { + gologger.Info().Msgf("Found %v results, View found results in dashboard : %v", u.counter.Load(), getAssetsDashBoardURL(u.assetGroupID)) + } + }() + // temporary buffer to store the results + buff := &bytes.Buffer{} + ticker := time.NewTicker(flushTimer) + + for { + select { + case <-ctx.Done(): + // flush before exit + if buff.Len() > 0 { + if err := u.uploadChunk(buff); err != nil { + gologger.Error().Msgf("Failed to upload scan results on cloud: %v", err) + } + } + return + case <-ticker.C: + // flush the buffer + if buff.Len() > 0 { + if err := u.uploadChunk(buff); err != nil { + gologger.Error().Msgf("Failed to upload scan results on cloud: %v", err) + } + } + case res, ok := <-u.data: + if !ok { + if buff.Len() > 0 { + if err := u.uploadChunk(buff); err != nil { + gologger.Error().Msgf("Failed to upload scan results on cloud: %v", err) + } + } + return + } + if res.Err != nil { + // skip we don't care + continue + } + lineBytes, err := json.Marshal(res) + if err != nil { + gologger.Error().Msgf("Failed to marshal result: %v", err) + continue + } + u.counter.Add(1) + line := conversion.String(lineBytes) + if buff.Len()+len(line) > MaxChunkSize { + // flush existing buffer + if err := u.uploadChunk(buff); err != nil { + gologger.Error().Msgf("Failed to upload asset results on cloud: %v", err) + } + } else { + buff.WriteString(line) + buff.WriteString("\n") + } + } + } +} + +// uploadChunk uploads a chunk of data to the server +func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error { + if err := u.upload(buff.Bytes()); err != nil { + return errorutil.NewWithErr(err).Msgf("could not upload chunk") + } + // if successful, reset the buffer + buff.Reset() + // log in verbose mode + gologger.Warning().Msgf("Uploaded results chunk, you can view assets at %v", getAssetsDashBoardURL(u.assetGroupID)) + return nil +} + +func (u *UploadWriter) upload(data []byte) error { + req, err := u.getRequest(data) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not create upload request") + } + resp, err := u.client.Do(req) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not upload results") + } + defer resp.Body.Close() + bin, err := io.ReadAll(resp.Body) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not get id from response") + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("could not upload results got status code %v on %v", resp.StatusCode, resp.Request.URL.String()) + } + var uploadResp uploadResponse + if err := json.Unmarshal(bin, &uploadResp); err != nil { + return errorutil.NewWithErr(err).Msgf("could not unmarshal response got %v", string(bin)) + } + if uploadResp.ID != "" && u.assetGroupID == "" { + u.assetGroupID = uploadResp.ID + } + return nil +} + +// getRequest returns a new request for upload +// if scanID is not provided create new scan by uploading the data +// if scanID is provided append the data to existing scan +func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { + var method, url string + + if u.assetGroupID == "" { + u.uploadURL.Path = uploadEndpoint + method = http.MethodPost + url = u.uploadURL.String() + } else { + u.uploadURL.Path = fmt.Sprintf(appendEndpoint, u.assetGroupID) + method = http.MethodPatch + url = u.uploadURL.String() + } + req, err := retryablehttp.NewRequest(method, url, bytes.NewReader(bin)) + if err != nil { + return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request") + } + // add pdtm meta params + req.URL.Params.Merge(updateutils.GetpdtmParams(runner.Version)) + // if it is upload endpoint also include name if it exists + if u.assetGroupName != "" && req.URL.Path == uploadEndpoint { + req.URL.Params.Add("name", u.assetGroupName) + } + req.URL.Update() + + req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) + if teamID != "" { + req.Header.Set(teamIDHeader, teamID) + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Accept", "application/json") + return req, nil +} + +// Close closes the upload writer +func (u *UploadWriter) Close() { + if !u.closed.Load() { + // protect to avoid channel closed twice error + close(u.data) + u.closed.Store(true) + } + <-u.done +} diff --git a/runner/banner.go b/runner/banner.go index 48c1e932..7f0b50e4 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -15,8 +15,8 @@ const banner = ` /_/ ` -// Version is the current version of httpx -const version = `v1.6.6` +// Version is the current Version of httpx +const Version = `v1.6.6` // showBanner is used to show the banner to the user func showBanner() { @@ -28,7 +28,7 @@ func showBanner() { func GetUpdateCallback() func() { return func() { showBanner() - updateutils.GetUpdateToolCallback("httpx", version)() + updateutils.GetUpdateToolCallback("httpx", Version)() } } diff --git a/runner/healthcheck.go b/runner/healthcheck.go index 42edbd77..c48490fb 100644 --- a/runner/healthcheck.go +++ b/runner/healthcheck.go @@ -14,7 +14,7 @@ func DoHealthCheck(options *Options, flagSet *goflags.FlagSet) string { // RW permissions on config file cfgFilePath, _ := flagSet.GetConfigFilePath() var test strings.Builder - test.WriteString(fmt.Sprintf("Version: %s\n", version)) + test.WriteString(fmt.Sprintf("Version: %s\n", Version)) test.WriteString(fmt.Sprintf("Operative System: %s\n", runtime.GOOS)) test.WriteString(fmt.Sprintf("Architecture: %s\n", runtime.GOARCH)) test.WriteString(fmt.Sprintf("Go Version: %s\n", runtime.Version())) diff --git a/runner/options.go b/runner/options.go index dff5cbe1..e168f5da 100644 --- a/runner/options.go +++ b/runner/options.go @@ -308,6 +308,17 @@ type Options struct { // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice Protocol string + // AssetUpload + AssetUpload bool + // AssetName + AssetName string + // AssetID + AssetID string + // AssetFileUpload + AssetFileUpload string + // OnClose adds a callback function that is invoked when httpx is closed + // to be exact at end of existing closures + OnClose func() // Optional pre-created objects to reduce allocations Wappalyzer *wappalyzer.Wappalyze @@ -440,7 +451,6 @@ func ParseOptions() *Options { flagSet.CreateGroup("configs", "Configurations", flagSet.StringVar(&cfgFile, "config", "", "path to the httpx configuration file (default $HOME/.config/httpx/config.yaml)"), - flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.NormalizedStringSliceOptions), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), @@ -493,6 +503,14 @@ func ParseOptions() *Options { flagSet.IntVarP(&options.MaxResponseBodySizeToRead, "response-size-to-read", "rstr", math.MaxInt32, "max response size to read in bytes"), ) + flagSet.CreateGroup("cloud", "Cloud", + flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), + flagSet.BoolVarP(&options.AssetUpload, "dashboard", "pd", false, "upload / view output in projectdiscovery cloud (pdcp) UI dashboard"), + flagSet.StringVarP(&options.AssetID, "asset-id", "aid", "", "upload new assets to existing asset id (optional)"), + flagSet.StringVarP(&options.AssetName, "asset-name", "aname", "", "assets group name to set (optional)"), + flagSet.StringVarP(&options.AssetFileUpload, "dashboard-upload", "pdu", "", "upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"), + ) + _ = flagSet.Parse() if options.OutputAll && options.Output == "" { @@ -561,18 +579,18 @@ func ParseOptions() *Options { showBanner() if options.Version { - gologger.Info().Msgf("Current Version: %s\n", version) + gologger.Info().Msgf("Current Version: %s\n", Version) os.Exit(0) } if !options.DisableUpdateCheck { - latestVersion, err := updateutils.GetToolVersionCallback("httpx", version)() + latestVersion, err := updateutils.GetToolVersionCallback("httpx", Version)() if err != nil { if options.Verbose { gologger.Error().Msgf("httpx version check failed: %v", err.Error()) } } else { - gologger.Info().Msgf("Current httpx version %v %v", version, updateutils.GetVersionDescription(version, latestVersion)) + gologger.Info().Msgf("Current httpx version %v %v", Version, updateutils.GetVersionDescription(Version, latestVersion)) } } diff --git a/runner/runner.go b/runner/runner.go index 4661c9e4..af3ffa22 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -113,7 +113,7 @@ func New(options *Options) (*Runner, error) { var err error if options.Wappalyzer != nil { runner.wappalyzer = options.Wappalyzer - } else if options.TechDetect || options.JSONOutput || options.CSVOutput { + } else if options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload { runner.wappalyzer, err = wappalyzer.New() } if err != nil { @@ -271,7 +271,7 @@ func New(options *Options) (*Runner, error) { scanopts.OutputResponseTime = options.OutputResponseTime scanopts.NoFallback = options.NoFallback scanopts.NoFallbackScheme = options.NoFallbackScheme - scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput + scanopts.TechDetect = options.TechDetect || options.JSONOutput || options.CSVOutput || options.AssetUpload scanopts.StoreChain = options.StoreChain scanopts.StoreVisionReconClusters = options.StoreVisionReconClusters scanopts.MaxResponseBodySizeToSave = options.MaxResponseBodySizeToSave @@ -672,6 +672,9 @@ func (r *Runner) Close() { if r.options.Screenshot { r.browser.Close() } + if r.options.OnClose != nil { + r.options.OnClose() + } } // RunEnumeration on targets for httpx client @@ -935,12 +938,6 @@ func (r *Runner) RunEnumeration() { continue } - // call the callback function if any - // be careful and check for result.Err - if r.options.OnResult != nil { - r.options.OnResult(resp) - } - // store responses or chain in directory URL, _ := urlutil.Parse(resp.URL) domainFile := resp.Method + ":" + URL.EscapedString() @@ -1102,6 +1099,12 @@ func (r *Runner) RunEnumeration() { plainFile.WriteString(resp.str + "\n") } + // call the callback function if any + // be careful and check for result.Err + if r.options.OnResult != nil { + r.options.OnResult(resp) + } + if r.options.JSONOutput { row := resp.JSON(&r.scanopts) @@ -2149,14 +2152,17 @@ retry: if scanopts.TechDetect && len(technologies) > 0 { sort.Strings(technologies) technologies := strings.Join(technologies, ",") - - builder.WriteString(" [") - if !scanopts.OutputWithNoColor { - builder.WriteString(aurora.Magenta(technologies).String()) - } else { - builder.WriteString(technologies) + // only print to console if tech-detect flag is enabled + // scanopts.TechDetect implicitly enabled for json , csv and asset-upload + if r.options.TechDetect { + builder.WriteString(" [") + if !scanopts.OutputWithNoColor { + builder.WriteString(aurora.Magenta(technologies).String()) + } else { + builder.WriteString(technologies) + } + builder.WriteRune(']') } - builder.WriteRune(']') } result := Result{ From 9b885a5d13d9e1f1935fe8ca090c933a5dc46db8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:06:29 +0000 Subject: [PATCH 14/21] chore(deps): bump github.com/projectdiscovery/retryablehttp-go Bumps [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) from 1.0.69 to 1.0.71. - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.69...v1.0.71) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 23781101..73b80009 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.0.20 github.com/projectdiscovery/dsl v0.1.7 - github.com/projectdiscovery/fastdialer v0.2.0 + github.com/projectdiscovery/fastdialer v0.2.1 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.60 @@ -34,7 +34,7 @@ require ( github.com/projectdiscovery/networkpolicy v0.0.9 github.com/projectdiscovery/ratelimit v0.0.45 github.com/projectdiscovery/rawhttp v0.1.57 - github.com/projectdiscovery/retryablehttp-go v1.0.69 + github.com/projectdiscovery/retryablehttp-go v1.0.71 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.60 github.com/projectdiscovery/utils v0.2.2 @@ -113,7 +113,7 @@ require ( github.com/projectdiscovery/freeport v0.0.5 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.67 // indirect + github.com/projectdiscovery/retryabledns v1.0.68 // indirect github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect diff --git a/go.sum b/go.sum index 4ba831ad..1acb0d88 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,8 @@ github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPo github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= github.com/projectdiscovery/dsl v0.1.7 h1:mPaHFPr2IMq69SjFGTt/DP8W5QeaGo+6lHqusnko2Qw= github.com/projectdiscovery/dsl v0.1.7/go.mod h1:pjBGO539itAgO+qD5PSXTcvRiF92tJKLzkA1aqCMwEQ= -github.com/projectdiscovery/fastdialer v0.2.0 h1:ZEYKA9L5VerrD9LcXH/gtTqUHiBQvTudrbqkZSRMpJo= -github.com/projectdiscovery/fastdialer v0.2.0/go.mod h1:bwBv51HzxK7DkCjB1EHQJKDzXKGPDLp6MgK8PPjaW/w= +github.com/projectdiscovery/fastdialer v0.2.1 h1:or3QuGW1jlZKi+IRkwxShSAG/hgR+yamd52RqjaZ28Q= +github.com/projectdiscovery/fastdialer v0.2.1/go.mod h1:FGPJZIPzAfR7SyDCPTsftaf61lGOqIjrJpwo2IgkNpg= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= @@ -252,10 +252,10 @@ github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSX github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko= github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE= -github.com/projectdiscovery/retryabledns v1.0.67 h1:PUrkoJaMRaoShMNwRg/cSDp5c1OjPlBYble23YAx8V0= -github.com/projectdiscovery/retryabledns v1.0.67/go.mod h1:2QDtjND4BkSECpMOs5fhgL0FYMk3UmGUnTwa9cqHe4I= -github.com/projectdiscovery/retryablehttp-go v1.0.69 h1:EbbHtZS1g/aT7sM3ZFNMcvnMfe9wuqXX2rEG1vGFKe4= -github.com/projectdiscovery/retryablehttp-go v1.0.69/go.mod h1:STJ0rpHJHofbAq22yI8nZCKTVWOk50xZ+oMJHL1rc5g= +github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA= +github.com/projectdiscovery/retryabledns v1.0.68/go.mod h1:72W9RwsHVRIGmtc4W6i6izVtYzKBTdnCE1VciqYM5Eg= +github.com/projectdiscovery/retryablehttp-go v1.0.71 h1:yXPNShCOwoTz7dBSJsBhBh4g4ujX62XS/BrH/fL1VyE= +github.com/projectdiscovery/retryablehttp-go v1.0.71/go.mod h1:wY3T89EwcCKAw6iyMDvwzGPyL3d8TaBU80hnDErEKgM= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= From f92aaa11225af5a3bdb13986003d6a2aabd08540 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:06:40 +0000 Subject: [PATCH 15/21] chore(deps): bump github.com/projectdiscovery/wappalyzergo Bumps [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) from 0.1.10 to 0.1.12. - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.1.10...v0.1.12) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 23781101..3e600148 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.60 github.com/projectdiscovery/utils v0.2.2 - github.com/projectdiscovery/wappalyzergo v0.1.10 + github.com/projectdiscovery/wappalyzergo v0.1.12 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 4ba831ad..38629da1 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2Z github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w= github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= -github.com/projectdiscovery/wappalyzergo v0.1.10 h1:cUTMw8xYXyKTAxUiGYltJxEh4OtJyCvaahj+RG3mHSY= -github.com/projectdiscovery/wappalyzergo v0.1.10/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/wappalyzergo v0.1.12 h1:usZmlyiNDRvdYstZpFQH11cx413Ppykhh5pg6zVvcY4= +github.com/projectdiscovery/wappalyzergo v0.1.12/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= From 64275c2fdaa49740a4765e3711cb42ed159e562c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:11:38 +0000 Subject: [PATCH 16/21] chore(deps): bump github.com/projectdiscovery/ratelimit Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.45 to 0.0.49. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.45...v0.0.49) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 73b80009..f85031c3 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/projectdiscovery/hmap v0.0.51 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/ratelimit v0.0.45 + github.com/projectdiscovery/ratelimit v0.0.49 github.com/projectdiscovery/rawhttp v0.1.57 github.com/projectdiscovery/retryablehttp-go v1.0.71 github.com/projectdiscovery/tlsx v1.1.6 diff --git a/go.sum b/go.sum index 1acb0d88..b3d2b2cc 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.45 h1:h28oF+hJ0CHcdBZozT1Go7ppWmzTxSXDKNNh2G1Ot9Q= -github.com/projectdiscovery/ratelimit v0.0.45/go.mod h1:1vSJUseDS7SjNwIBi9wNRcgsMKNTLxy/GfdlLFVbgI4= +github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ= +github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k= github.com/projectdiscovery/rawhttp v0.1.57 h1:2vCT2i1NSZbTBH+uUBrxOJjxDPKgIl2q6BGtQjs/Hko= github.com/projectdiscovery/rawhttp v0.1.57/go.mod h1:qtthyaU0k8eqcEdza1R/fTqwyxSK4BZ511ThxgkiQtE= github.com/projectdiscovery/retryabledns v1.0.68 h1:EWWG7WsGTT0YvwIjHclIWSWgv4R29xMWShR6Yt5Z+pA= From cdd96ea60d9a0e62d7c52147f145041fb7a7d503 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:14:32 +0000 Subject: [PATCH 17/21] chore(deps): bump github.com/projectdiscovery/utils from 0.2.2 to 0.2.3 Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.2.2 to 0.2.3. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.2.2...v0.2.3) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 372a04d7..8d215f97 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/projectdiscovery/retryablehttp-go v1.0.71 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/useragent v0.0.60 - github.com/projectdiscovery/utils v0.2.2 + github.com/projectdiscovery/utils v0.2.3 github.com/projectdiscovery/wappalyzergo v0.1.12 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 diff --git a/go.sum b/go.sum index a39c2516..0b56abfc 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,8 @@ github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1Uzc github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= github.com/projectdiscovery/useragent v0.0.60 h1:qDU1rwA+XOKmSqp7yoijAN4PuvLQc2ZvuaoWH7QIkuc= github.com/projectdiscovery/useragent v0.0.60/go.mod h1:05IDiJEy2dWl3x6dnsWtJYPwT40oWha144Us7+Fwr6w= -github.com/projectdiscovery/utils v0.2.2 h1:ZzanMTiVboM7vFO6cjt+GRDtvVjZD1GChBa7knekBgs= -github.com/projectdiscovery/utils v0.2.2/go.mod h1:sZHBCrS37ejIb0/pFjKZewm1nBWMTXUsLJH3hTVww0Y= +github.com/projectdiscovery/utils v0.2.3 h1:rkambl0EoTF/y6DpjCfSwcVUFdkAeVOtYkK3lX6InCY= +github.com/projectdiscovery/utils v0.2.3/go.mod h1:eGuuQ5Acekg47WsFS1Q9Qxw8+vI6IxwqIQSAplBBG0c= github.com/projectdiscovery/wappalyzergo v0.1.12 h1:usZmlyiNDRvdYstZpFQH11cx413Ppykhh5pg6zVvcY4= github.com/projectdiscovery/wappalyzergo v0.1.12/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= From 068b3070faf8a4e5946f40ac1a525db1474a2570 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:18:40 +0000 Subject: [PATCH 18/21] chore(deps): bump github.com/projectdiscovery/gologger Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.16 to 1.1.17. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.16...v1.1.17) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 52f5089a..675a9efc 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 github.com/projectdiscovery/goflags v0.1.60 - github.com/projectdiscovery/gologger v1.1.16 + github.com/projectdiscovery/gologger v1.1.17 github.com/projectdiscovery/hmap v0.0.51 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 diff --git a/go.sum b/go.sum index 5b4f862a..17faafe3 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQ github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= github.com/projectdiscovery/goflags v0.1.60 h1:S1PtMHdEOFqb8CjkXuaomtqUptdApuV/IF2Wvn4QGZw= github.com/projectdiscovery/goflags v0.1.60/go.mod h1:8pKusnHPcnPW6WGDRG9auCZ6RH9twyDp3aETsijF73A= -github.com/projectdiscovery/gologger v1.1.16 h1:NsYQVNt1H8O3Wmag4sXxQxvbLbIP16bjCm/cnCvf5hc= -github.com/projectdiscovery/gologger v1.1.16/go.mod h1:WlyfroigIqU/in8A3fTEeMJ6t5NfbCG+rgWcvI5dQiQ= +github.com/projectdiscovery/gologger v1.1.17 h1:4KWUgXGKLzXqyDAl9A+fiCcgGgHa4zo7++166xDRSqo= +github.com/projectdiscovery/gologger v1.1.17/go.mod h1:JAo10tJopMAUHDdKTX/ZSb1QC/sgxfr5ovrd+SAt7fM= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/hmap v0.0.51 h1:xqbpRAJRHPMoS2uERkbWGObIO4bv+whe3PEk3h4lDEg= From fe00d47fbf86f95243cb5e8b8d8e5ce468919cb1 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 30 Jul 2024 19:43:17 +0300 Subject: [PATCH 19/21] introduce path flag for `-fep` (#1830) Co-authored-by: Sandeep Singh --- README.md | 37 +++++++++++++++++++------------------ runner/options.go | 2 ++ runner/runner.go | 17 +++++++++++++---- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6fa81afc..c94f7418 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string[] match response with specified string (-ms admin) -mr, -match-regex string[] match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google) + -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition @@ -151,7 +151,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) -fs, -filter-string string[] filter response with specified string (-fs admin) -fe, -filter-regex string[] filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (leaseweb, stackpath, cloudfront, fastly, google) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition -strip strips all tags in response. supported formats: html,xml (default html) @@ -178,21 +178,22 @@ UPDATE: -duc, -disable-update-check disable automatic httpx update check OUTPUT: - -o, -output string file to write output results - -oa, -output-all filename to write output results in all formats - -sr, -store-response store http response to output directory - -srd, -store-response-dir string store http response to custom directory - -ob, -omit-body omit response body in output - -csv store output in csv format - -csvo, -csv-output-encoding string define output encoding - -j, -json store output in JSONL(ines) format - -irh, -include-response-header include http response (headers) in JSON output (-json only) - -irr, -include-response include http request/response (headers + body) in JSON output (-json only) - -irrb, -include-response-base64 include base64 encoded http request/response in JSON output (-json only) - -include-chain include redirect http chain in JSON output (-json only) - -store-chain include http redirect chain in responses (-sr only) - -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only) - -pr, -protocol string protocol to use (unknown, http11) + -o, -output string file to write output results + -oa, -output-all filename to write output results in all formats + -sr, -store-response store http response to output directory + -srd, -store-response-dir string store http response to custom directory + -ob, -omit-body omit response body in output + -csv store output in csv format + -csvo, -csv-output-encoding string define output encoding + -j, -json store output in JSONL(ines) format + -irh, -include-response-header include http response (headers) in JSON output (-json only) + -irr, -include-response include http request/response (headers + body) in JSON output (-json only) + -irrb, -include-response-base64 include base64 encoded http request/response in JSON output (-json only) + -include-chain include redirect http chain in JSON output (-json only) + -store-chain include http redirect chain in responses (-sr only) + -svrc, -store-vision-recon-cluster include visual recon clusters (-ss and -sr only) + -pr, -protocol string protocol to use (unknown, http11) + -fepp, -filter-error-page-path string path to store filtered error pages (default "filtered_error_page.json") CONFIGURATIONS: -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) @@ -237,7 +238,7 @@ DEBUG: OPTIMIZATIONS: -nf, -no-fallback display both probed protocol (HTTPS and HTTP) - -nfs, -no-fallback-scheme probe with protocol scheme specified in input + -nfs, -no-fallback-scheme probe with protocol scheme specified in input -maxhr, -max-host-error int max error count per host before skipping remaining path/s (default 30) -e, -exclude string[] exclude host matching specified filter ('cdn', 'private-ips', cidr, ip, regex) -retries int number of retries diff --git a/runner/options.go b/runner/options.go index e168f5da..69c67a62 100644 --- a/runner/options.go +++ b/runner/options.go @@ -308,6 +308,7 @@ type Options struct { // HeadlessOptionalArguments specifies optional arguments to pass to Chrome HeadlessOptionalArguments goflags.StringSlice Protocol string + OutputFilterErrorPagePath string // AssetUpload AssetUpload bool // AssetName @@ -447,6 +448,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.StoreChain, "store-chain", false, "include http redirect chain in responses (-sr only)"), flagSet.BoolVarP(&options.StoreVisionReconClusters, "store-vision-recon-cluster", "svrc", false, "include visual recon clusters (-ss and -sr only)"), flagSet.StringVarP(&options.Protocol, "protocol", "pr", "", "protocol to use (unknown, http11)"), + flagSet.StringVarP(&options.OutputFilterErrorPagePath, "filter-error-page-path", "fepp", "filtered_error_page.json", "path to store filtered error pages"), ) flagSet.CreateGroup("configs", "Configurations", diff --git a/runner/runner.go b/runner/runner.go index af3ffa22..06ea14f2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -868,7 +868,7 @@ func (r *Runner) RunEnumeration() { } if r.options.OutputFilterErrorPage && resp.KnowledgeBase["PageType"] == "error" { - logFilteredErrorPage(resp.URL) + logFilteredErrorPage(r.options.OutputFilterErrorPagePath, resp.URL) continue } if len(r.options.filterStatusCode) > 0 && sliceutil.Contains(r.options.filterStatusCode, resp.StatusCode) { @@ -1251,9 +1251,17 @@ func (r *Runner) RunEnumeration() { } } -func logFilteredErrorPage(url string) { - fileName := "filtered_error_page.json" - file, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600) +func logFilteredErrorPage(fileName, url string) { + dir := filepath.Dir(fileName) + if !fileutil.FolderExists(dir) { + err := fileutil.CreateFolder(dir) + if err != nil { + gologger.Fatal().Msgf("Could not create directory '%s': %s\n", dir, err) + return + } + } + + file, err := fileutil.OpenOrCreateFile(fileName) if err != nil { gologger.Fatal().Msgf("Could not open/create output file '%s': %s\n", fileName, err) return @@ -1281,6 +1289,7 @@ func logFilteredErrorPage(url string) { return } } + func openOrCreateFile(resume bool, filename string) *os.File { var err error var f *os.File From dfe27851f5a457fb4e4825e663b2f4268fc9d611 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 1 Aug 2024 00:09:47 +0530 Subject: [PATCH 20/21] version update --- runner/banner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner/banner.go b/runner/banner.go index 7f0b50e4..ad38d311 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -16,7 +16,7 @@ const banner = ` ` // Version is the current Version of httpx -const Version = `v1.6.6` +const Version = `v1.6.7` // showBanner is used to show the banner to the user func showBanner() { From 56f4474f30b1838edfc6ea580a4474df7160bf66 Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Thu, 1 Aug 2024 00:12:59 +0530 Subject: [PATCH 21/21] updated docs to include dashboard options --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c94f7418..db46be5b 100644 --- a/README.md +++ b/README.md @@ -134,13 +134,13 @@ MATCHERS: -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) -ms, -match-string string[] match response with specified string (-ms admin) -mr, -match-regex string[] match response with specified regex (-mr admin) - -mcdn, -match-cdn string[] match host with specified cdn provider (cloudfront, fastly, google) + -mcdn, -match-cdn string[] match host with specified cdn provider (google, cloudfront, fastly) -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') -mdc, -match-condition string match response with dsl expression condition EXTRACTOR: -er, -extract-regex string[] display response content with matched regex - -ep, -extract-preset string[] display response content matched by a pre-defined regex (url,ipv4,mail) + -ep, -extract-preset string[] display response content matched by a pre-defined regex (ipv4,mail,url) FILTERS: -fc, -filter-code string filter response with specified status code (-fc 403,401) @@ -151,7 +151,7 @@ FILTERS: -ffc, -filter-favicon string[] filter response with specified favicon hash (-ffc 1494302000) -fs, -filter-string string[] filter response with specified string (-fs admin) -fe, -filter-regex string[] filter response with specified regex (-fe admin) - -fcdn, -filter-cdn string[] filter host with specified cdn provider (cloudfront, fastly, google) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, cloudfront, fastly) -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') -fdc, -filter-condition string filter response with dsl expression condition -strip strips all tags in response. supported formats: html,xml (default html) @@ -197,7 +197,6 @@ OUTPUT: CONFIGURATIONS: -config string path to the httpx configuration file (default $HOME/.config/httpx/config.yaml) - -auth configure projectdiscovery cloud (pdcp) api key (default true) -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) @@ -246,6 +245,13 @@ OPTIMIZATIONS: -delay value duration between each http request (eg: 200ms, 1s) (default -1ns) -rsts, -response-size-to-save int max response size to save in bytes (default 2147483647) -rstr, -response-size-to-read int max response size to read in bytes (default 2147483647) + +CLOUD: + -auth configure projectdiscovery cloud (pdcp) api key (default true) + -pd, -dashboard upload / view output in projectdiscovery cloud (pdcp) UI dashboard + -aid, -asset-id string upload new assets to existing asset id (optional) + -aname, -asset-name string assets group name to set (optional) + -pdu, -dashboard-upload string upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard ``` # Running httpx