Merge pull request #466 from epi052/add-redirect-messages-to-normal-r…

…eports

Add redirect messages to normal reports

.github/pull_request_template.md

@@ -16,7 +16,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Documentation
 - [ ] New code is documented using [doc comments](https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html)
-- [ ] Documentation about your PR is included in the README, as needed
+- [ ] Documentation about your PR is included in the `docs`, as needed. The docs live in a [separate repository](https://epi052.github.io/feroxbuster-docs/docs/). Update the appropriate pages at the links below.
+  - [ ] update [example config file section](https://epi052.github.io/feroxbuster-docs/docs/configuration/ferox-config-toml/)
+  - [ ] update [help output section](https://epi052.github.io/feroxbuster-docs/docs/configuration/command-line/)
+  - [ ] add an [example](https://epi052.github.io/feroxbuster-docs/docs/examples/)
+  - [ ] update [comparison table](https://epi052.github.io/feroxbuster-docs/docs/compare/)
 
 ## Additional Tests
 - [ ] New code is unit tested

build.rs

@@ -16,20 +16,11 @@ fn main() {
 
     let mut app = initialize();
 
-    let path = generate_to(shells::Bash, &mut app, "feroxbuster", outdir).unwrap();
-    println!("cargo:warning=completion file is generated: {path:?}");
-
-    let path = generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
-    println!("cargo:warning=completion file is generated: {path:?}");
-
-    let path = generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
-    println!("cargo:warning=completion file is generated: {path:?}");
-
-    let path = generate_to(shells::PowerShell, &mut app, "feroxbuster", outdir).unwrap();
-    println!("cargo:warning=completion file is generated: {path:?}");
-
-    let path = generate_to(shells::Elvish, &mut app, "feroxbuster", outdir).unwrap();
-    println!("cargo:warning=completion file is generated: {path:?}");
+    generate_to(shells::Bash, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::PowerShell, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Elvish, &mut app, "feroxbuster", outdir).unwrap();
 
     // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
     // landed on was to add -o plusdirs to the bash completion script. The following code aims to

src/response.rs

@@ -7,6 +7,7 @@ use std::{
 };
 
 use anyhow::{Context, Result};
+use console::style;
 use reqwest::{
     header::{HeaderMap, HeaderName, HeaderValue},
     Method, Response, StatusCode, Url,
@@ -351,6 +352,30 @@ impl FeroxSerialize for FeroxResponse {
         let method = self.method().as_str();
         let wild_status = status_colorizer("WLD");
 
+        let mut url_with_redirect = match (
+            self.status().is_redirection(),
+            self.headers().get("Location").is_some(),
+        ) {
+            (true, true) => {
+                // redirect with Location header, show where it goes if possible
+                let loc = self
+                    .headers()
+                    .get("Location")
+                    .unwrap() // known Some() already
+                    .to_str()
+                    .unwrap_or("Unknown");
+
+                // prettify the redirect target
+                let loc = style(loc).yellow();
+
+                format!("{} => {loc}", self.url())
+            }
+            _ => {
+                // no redirect, just use the normal url
+                self.url().to_string()
+            }
+        };
+
         if self.wildcard && matches!(self.output_level, OutputLevel::Default | OutputLevel::Quiet) {
             // --silent was not used and response is a wildcard, special messages abound when
             // this is the case...
@@ -369,25 +394,16 @@ impl FeroxSerialize for FeroxResponse {
             );
 
             if self.status().is_redirection() {
-                // when it's a redirect, show where it goes, if possible
-                if let Some(next_loc) = self.headers().get("Location") {
-                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-
-                    let redirect_msg = format!(
-                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
-                        wild_status,
-                        "-",
-                        "-",
-                        "-",
-                        self.url(),
-                        next_loc_str
-                    );
-
-                    message.push_str(&redirect_msg);
-                }
+                // initial wildcard messages are wordy enough, put the redirect by itself
+                url_with_redirect = format!(
+                    "{} {:>9} {:>9} {:>9} {}\n",
+                    wild_status, "-", "-", "-", url_with_redirect
+                );
+
+                // base message + redirection message (either empty string or redir msg)
+                message.push_str(&url_with_redirect);
             }
 
-            // base message + redirection message (if appropriate)
             message
         } else {
             // not a wildcard, just create a normal entry
@@ -397,7 +413,7 @@ impl FeroxSerialize for FeroxResponse {
                 &lines,
                 &words,
                 &chars,
-                self.url().as_str(),
+                &url_with_redirect,
                 self.output_level,
             )
         }

src/scanner/ferox_scanner.rs

@@ -74,7 +74,7 @@ impl FeroxScanner {
         log::trace!("enter: scan_url");
         log::info!("Starting scan against: {}", self.target_url);
 
-        let scan_timer = Instant::now();
+        let mut scan_timer = Instant::now();
         let mut dirlist_flag = false;
 
         if self.handles.config.extract_links {
@@ -133,7 +133,10 @@ impl FeroxScanner {
             ))?;
 
             progress_bar.reset_eta();
-            progress_bar.finish_with_message(&format!("=> {}", style("Directory listing").green()));
+            progress_bar.finish_with_message(&format!(
+                "=> {}",
+                style("Directory listing").blue().bright()
+            ));
 
             ferox_scan.finish()?;
 
@@ -145,6 +148,10 @@ impl FeroxScanner {
         // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
         // to the caller.
         let _permit = self.scan_limiter.acquire().await;
+        if self.handles.config.scan_limit > 0 {
+            scan_timer = Instant::now();
+            progress_bar.reset();
+        }
 
         // Arc clones to be passed around to the various scans
         let looping_words = self.wordlist.clone();

src/scanner/requester.rs

@@ -57,7 +57,6 @@ pub(super) struct Requester {
     /// need a usize to determine the number of consecutive non-error calls that a requester has
     /// seen; this will satisfy the non-mut self constraint (due to us being behind an Arc, and
     /// the need for a counter)
-    #[allow(clippy::mutex_atomic)]
     tuning_lock: Mutex<usize>,
 }
 

src/utils.rs

@@ -168,31 +168,28 @@ pub async fn make_request(
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
-                    let fancy_message = format!("{} !=> {}", url, last_redirect);
-
-                    let report = if let Some(msg_status) = e.status() {
-                        send_command!(tx_stats, AddStatus(msg_status));
-                        create_report_string(
-                            msg_status.as_str(),
-                            method,
-                            "-1",
-                            "-1",
-                            "-1",
-                            &fancy_message,
-                            output_level,
-                        )
-                    } else {
-                        create_report_string(
-                            "UNK",
-                            method,
-                            "-1",
-                            "-1",
-                            "-1",
-                            &fancy_message,
-                            output_level,
-                        )
+                    let fancy_message = format!(
+                        "{} !=> {} ({})",
+                        url,
+                        last_redirect,
+                        style("too many redirects").red(),
+                    );
+
+                    let msg_status = match e.status() {
+                        Some(status) => status.to_string(),
+                        None => "ERR".to_string(),
                     };
 
+                    let report = create_report_string(
+                        &msg_status,
+                        method,
+                        "-1",
+                        "-1",
+                        "-1",
+                        &fancy_message,
+                        output_level,
+                    );
+
                     send_command!(tx_stats, AddError(Redirection));
 
                     ferox_print(&report, &PROGRESS_PRINTER)

tests/test_heuristics.rs

@@ -454,12 +454,12 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
     assert!(contents.contains("WLD"));
     assert!(contents.contains("301"));
     assert!(contents.contains("/some-redirect"));
-    assert!(contents.contains("redirects to => "));
+    assert!(contents.contains(" => "));
     assert!(contents.contains(&srv.url("/")));
     assert!(contents.contains("(url length: 32)"));
 
     cmd.assert().success().stdout(
-        predicate::str::contains("redirects to => ")
+        predicate::str::contains(" => ")
             .and(predicate::str::contains("/some-redirect"))
             .and(predicate::str::contains("301"))
             .and(predicate::str::contains(srv.url("/")))