From e6f7a00ba0a3c6f55a399e3f55c769bbc9c3d0bb Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 14 Nov 2020 10:11:05 -0600 Subject: [PATCH 1/7] initial guess at grabbing the correct info --- src/reporter.rs | 22 ++++++---------------- src/utils.rs | 26 +++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/reporter.rs b/src/reporter.rs index b87152f7..8a295fb9 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -1,5 +1,5 @@ use crate::config::{CONFIGURATION, PROGRESS_PRINTER}; -use crate::utils::{ferox_print, make_request, status_colorizer}; +use crate::utils::{create_report_string, ferox_print, make_request, status_colorizer}; use crate::{FeroxChannel, FeroxResponse}; use console::strip_ansi_codes; use std::io::Write; @@ -95,21 +95,11 @@ async fn spawn_terminal_reporter( log::trace!("received {} on reporting channel", resp.url()); if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) { - let report = if CONFIGURATION.quiet { - // -q used, just need the url - format!("{}\n", resp.url()) - } else { - // normal printing with status and size - let status = status_colorizer(&resp.status().as_str()); - format!( - // example output - // 200 3280 https://localhost.com/FAQ - "{} {:>10} {}\n", - status, - resp.content_length(), - resp.url() - ) - }; + let report = create_report_string( + resp.status().as_str(), + &resp.content_length().to_string(), + &resp.url().to_string(), + ); // print to stdout ferox_print(&report, &PROGRESS_PRINTER); diff --git a/src/utils.rs b/src/utils.rs index e70a8f87..d4d3e815 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,4 @@ -use crate::{FeroxError, FeroxResult}; +use crate::{config::{CONFIGURATION, PROGRESS_PRINTER}, FeroxError, FeroxResult}; use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; use reqwest::Url; @@ -253,6 +253,15 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { if e.to_string().contains("operation timed out") { // only warn for timeouts, while actual errors are still left as errors log::warn!("Error while making request: {}", e); + } else if e.is_redirect() { + let last_redirect = e.url().unwrap(); + let fancy_message = format!("{} !=> {}", url, last_redirect); + let report = create_report_string( + e.status().unwrap().as_str(), + "-1", + &fancy_message, + ); + ferox_print(&report, &PROGRESS_PRINTER) } else { log::error!("Error while making request: {}", e); } @@ -261,6 +270,21 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { } } +/// Helper to create the standard line for output to file/terminal +/// +/// example output: +/// 200 3280 https://localhost.com/FAQ +pub fn create_report_string(status: &str, content_length: &str, url: &str) -> String { + if CONFIGURATION.quiet { + // -q used, just need the url + format!("{}\n", url) + } else { + // normal printing with status and size + let color_status = status_colorizer(status); + format!("{} {:>10} {}\n", color_status, content_length, url) + } +} + /// Attempts to set the soft limit for the RLIMIT_NOFILE resource /// /// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process From 88260e0b0428c861f9cb53b2e782e4e2c402644f Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 14 Nov 2020 15:34:18 -0600 Subject: [PATCH 2/7] toned down logging --- src/filters.rs | 4 ++-- src/heuristics.rs | 2 +- src/lib.rs | 7 +++++++ src/logger.rs | 8 ++++---- src/parser.rs | 2 +- src/scanner.rs | 20 ++++++-------------- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/filters.rs b/src/filters.rs index 12442156..6b5e5a4f 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -53,7 +53,7 @@ impl FeroxFilter for WildcardFilter { /// Examine size, dynamic, and content_len to determine whether or not the response received /// is a wildcard response and therefore should be filtered out fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {:?})", self, response); + log::trace!("enter: should_filter_response({:?} {})", self, response); // quick return if dont_filter is set if CONFIGURATION.dont_filter { @@ -114,7 +114,7 @@ pub struct StatusCodeFilter { impl FeroxFilter for StatusCodeFilter { /// Check `filter_code` against what was passed in via -C|--filter-status fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {:?})", self, response); + log::trace!("enter: should_filter_response({:?} {})", self, response); if response.status().as_u16() == self.filter_code { log::debug!( diff --git a/src/heuristics.rs b/src/heuristics.rs index 8675f9db..260a3764 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -229,7 +229,7 @@ async fn make_wildcard_request( } } } - log::trace!("exit: make_wildcard_request -> {:?}", ferox_response); + log::trace!("exit: make_wildcard_request -> {}", ferox_response); return Some(ferox_response); } } diff --git a/src/lib.rs b/src/lib.rs index 8051d86f..518bf7e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,6 +103,13 @@ pub struct FeroxResponse { headers: HeaderMap, } +/// Implement Display for FeroxResponse +impl fmt::Display for FeroxResponse { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "FeroxResponse {{ url: {}, status: {}, content-length: {} }}", self.url(), self.status(), self.content_length()) + } +} + /// `FeroxResponse` implementation impl FeroxResponse { /// Get the `StatusCode` of this `FeroxResponse` diff --git a/src/logger.rs b/src/logger.rs index f3bc726f..c8fc9026 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -19,8 +19,8 @@ pub fn initialize(verbosity: u8) { 0 => (), 1 => env::set_var("RUST_LOG", "warn"), 2 => env::set_var("RUST_LOG", "info"), - 3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"), - _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"), + 3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"), + _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"), } } } @@ -55,12 +55,12 @@ pub fn initialize(verbosity: u8) { }; let msg = format!( - "{} {:10.03} {}\n", + "{} {:10.03} {} {}\n", style(level_name).bg(level_color).black(), style(t).dim(), + record.target(), style(record.args()).dim(), ); - PROGRESS_PRINTER.println(&msg); if let Some(buffered_file) = locked_file.clone() { diff --git a/src/parser.rs b/src/parser.rs index ab545d4e..e4fc9e7b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -55,7 +55,7 @@ pub fn initialize() -> App<'static, 'static> { .long("verbosity") .takes_value(false) .multiple(true) - .help("Increase verbosity level (use -vv or more for greater effect)"), + .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"), ) .arg( Arg::with_name("proxy") diff --git a/src/scanner.rs b/src/scanner.rs index eb727aa3..9b73e226 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -285,7 +285,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx) /// or if the Location header is present and matches the base url + / (3xx) fn response_is_directory(response: &FeroxResponse) -> bool { - log::trace!("enter: is_directory({:?})", response); + log::trace!("enter: is_directory({})", response); if response.status().is_redirection() { // status code is 3xx @@ -312,7 +312,7 @@ fn response_is_directory(response: &FeroxResponse) -> bool { } None => { log::debug!( - "expected Location header, but none was found: {:?}", + "expected Location header, but none was found: {}", response ); log::trace!("exit: is_directory -> false"); @@ -370,7 +370,7 @@ async fn try_recursion( transmitter: UnboundedSender, ) { log::trace!( - "enter: try_recursion({:?}, {}, {:?})", + "enter: try_recursion({}, {}, {:?})", response, base_depth, transmitter @@ -523,11 +523,7 @@ async fn make_requests( if new_ferox_response.is_file() { // very likely a file, simply request and report - log::debug!( - "Singular extraction: {} ({})", - new_ferox_response.url(), - new_ferox_response.status().as_str(), - ); + log::debug!("Singular extraction: {}", new_ferox_response); send_report(report_chan.clone(), new_ferox_response); @@ -535,11 +531,7 @@ async fn make_requests( } if !CONFIGURATION.no_recursion { - log::debug!( - "Recursive extraction: {} ({})", - new_ferox_response.url(), - new_ferox_response.status().as_str() - ); + log::debug!("Recursive extraction: {}", new_ferox_response); if new_ferox_response.status().is_success() && !new_ferox_response.url().as_str().ends_with('/') @@ -565,7 +557,7 @@ async fn make_requests( /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel` fn send_report(report_sender: UnboundedSender, response: FeroxResponse) { - log::trace!("enter: send_report({:?}, {:?}", report_sender, response); + log::trace!("enter: send_report({:?}, {}", report_sender, response); match report_sender.send(response) { Ok(_) => {} From 23279eb1ed18fa0ad1c300812fc06e66e0581832 Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 14 Nov 2020 15:49:42 -0600 Subject: [PATCH 3/7] removed debug message that just reported the url --- src/utils.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.rs b/src/utils.rs index d4d3e815..399f8442 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -244,7 +244,6 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { match client.get(url.to_owned()).send().await { Ok(resp) => { - log::debug!("requested Url: {}", resp.url()); log::trace!("exit: make_request -> {:?}", resp); Ok(resp) } From 48e53be24409357121a6c0c8f377bd2366340540 Mon Sep 17 00:00:00 2001 From: epi Date: Sun, 15 Nov 2020 06:37:39 -0600 Subject: [PATCH 4/7] cleaned up make_request, ran fmt --- src/lib.rs | 8 +++++++- src/logger.rs | 1 + src/reporter.rs | 2 +- src/scanner.rs | 5 +---- src/utils.rs | 28 +++++++++++++++++----------- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 518bf7e8..a879c2f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -106,7 +106,13 @@ pub struct FeroxResponse { /// Implement Display for FeroxResponse impl fmt::Display for FeroxResponse { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "FeroxResponse {{ url: {}, status: {}, content-length: {} }}", self.url(), self.status(), self.content_length()) + write!( + f, + "FeroxResponse {{ url: {}, status: {}, content-length: {} }}", + self.url(), + self.status(), + self.content_length() + ) } } diff --git a/src/logger.rs b/src/logger.rs index c8fc9026..3c681a3a 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -61,6 +61,7 @@ pub fn initialize(verbosity: u8) { record.target(), style(record.args()).dim(), ); + PROGRESS_PRINTER.println(&msg); if let Some(buffered_file) = locked_file.clone() { diff --git a/src/reporter.rs b/src/reporter.rs index 8a295fb9..f7db38fb 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -1,5 +1,5 @@ use crate::config::{CONFIGURATION, PROGRESS_PRINTER}; -use crate::utils::{create_report_string, ferox_print, make_request, status_colorizer}; +use crate::utils::{create_report_string, ferox_print, make_request}; use crate::{FeroxChannel, FeroxResponse}; use console::strip_ansi_codes; use std::io::Write; diff --git a/src/scanner.rs b/src/scanner.rs index 9b73e226..96b4058e 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -311,10 +311,7 @@ fn response_is_directory(response: &FeroxResponse) -> bool { } } None => { - log::debug!( - "expected Location header, but none was found: {}", - response - ); + log::debug!("expected Location header, but none was found: {}", response); log::trace!("exit: is_directory -> false"); return false; } diff --git a/src/utils.rs b/src/utils.rs index 399f8442..f9519921 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,8 +1,10 @@ -use crate::{config::{CONFIGURATION, PROGRESS_PRINTER}, FeroxError, FeroxResult}; +use crate::{ + config::{CONFIGURATION, PROGRESS_PRINTER}, + FeroxError, FeroxResult, +}; use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; -use reqwest::Url; -use reqwest::{Client, Response}; +use reqwest::{Url, Client, Response}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource, Rlim}; use std::convert::TryInto; @@ -253,14 +255,18 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { // only warn for timeouts, while actual errors are still left as errors log::warn!("Error while making request: {}", e); } else if e.is_redirect() { - let last_redirect = e.url().unwrap(); - let fancy_message = format!("{} !=> {}", url, last_redirect); - let report = create_report_string( - e.status().unwrap().as_str(), - "-1", - &fancy_message, - ); - ferox_print(&report, &PROGRESS_PRINTER) + if let Some(last_redirect) = e.url() { + // get where we were headed (last_redirect) and where we came from (url) + let fancy_message = format!("{} !=> {}", url, last_redirect); + + let report = if let Some(msg_status) = e.status() { + create_report_string(msg_status.as_str(), "-1", &fancy_message) + } else { + create_report_string("UNK", "-1", &fancy_message) + }; + + ferox_print(&report, &PROGRESS_PRINTER) + }; } else { log::error!("Error while making request: {}", e); } From 771a9556f1531b2f5da8391962790e0160917ca7 Mon Sep 17 00:00:00 2001 From: epi Date: Sun, 15 Nov 2020 06:39:02 -0600 Subject: [PATCH 5/7] cleaned up make_request, ran fmt --- src/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index f9519921..572710b5 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,7 +4,7 @@ use crate::{ }; use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; -use reqwest::{Url, Client, Response}; +use reqwest::{Client, Response, Url}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource, Rlim}; use std::convert::TryInto; @@ -260,7 +260,7 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { let fancy_message = format!("{} !=> {}", url, last_redirect); let report = if let Some(msg_status) = e.status() { - create_report_string(msg_status.as_str(), "-1", &fancy_message) + create_report_string(msg_status.as_str(), "-1", &fancy_message) } else { create_report_string("UNK", "-1", &fancy_message) }; From 4b811a42b94b0a4b9b334cfd884dcf985607e73d Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 17 Nov 2020 17:22:03 -0600 Subject: [PATCH 6/7] tidied up a few report strings and fixed a clippy issue --- src/client.rs | 45 +++++++++++++++++++++++---------------------- src/heuristics.rs | 8 ++++++-- src/reporter.rs | 2 ++ src/utils.rs | 21 +++++++++++++++------ 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/client.rs b/src/client.rs index 7727dd48..eec7e336 100644 --- a/src/client.rs +++ b/src/client.rs @@ -32,31 +32,32 @@ pub fn initialize( .default_headers(header_map) .redirect(policy); - let client = if proxy.is_some() && !proxy.unwrap().is_empty() { - match Proxy::all(proxy.unwrap()) { - Ok(proxy_obj) => client.proxy(proxy_obj), - Err(e) => { - eprintln!( - "{} {} Could not add proxy ({:?}) to Client configuration", - status_colorizer("ERROR"), - module_colorizer("Client::initialize"), - proxy - ); - eprintln!( - "{} {} {}", - status_colorizer("ERROR"), - module_colorizer("Client::initialize"), - e - ); + let client = match proxy { + Some(some_proxy) => { + if !some_proxy.is_empty() { + // it's not an empty string + match Proxy::all(some_proxy) { + Ok(proxy_obj) => client.proxy(proxy_obj), + Err(e) => { + eprintln!( + "{} {} {}", + status_colorizer("ERROR"), + module_colorizer("Client::initialize"), + e + ); - #[cfg(test)] - panic!(); - #[cfg(not(test))] - exit(1); + #[cfg(test)] + panic!(); + #[cfg(not(test))] + exit(1); + } + } + } else { + client // Some("") was used? } } - } else { - client + // a proxy is specified, just need to add it to the client + None => client, }; match client.build() { diff --git a/src/heuristics.rs b/src/heuristics.rs index ea65a0ed..7b53ab26 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -89,8 +89,10 @@ pub async fn wildcard_test( if !CONFIGURATION.quiet { let msg = format!( - "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", + "{} {:>8}l {:>8}w {:>8}c Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", status_colorizer("WLD"), + ferox_response.line_count(), + ferox_response.word_count(), wildcard.dynamic, style("auto-filtering").yellow(), style(wc_length - url_len).cyan(), @@ -110,8 +112,10 @@ pub async fn wildcard_test( if !CONFIGURATION.quiet { let msg = format!( - "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", + "{} {:>8}l {:>8}w {:>8}c Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", status_colorizer("WLD"), + ferox_response.line_count(), + ferox_response.word_count(), wc_length, style("auto-filtering").yellow(), style(wc_length).cyan(), diff --git a/src/reporter.rs b/src/reporter.rs index f7db38fb..09d94070 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -97,6 +97,8 @@ async fn spawn_terminal_reporter( if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) { let report = create_report_string( resp.status().as_str(), + &resp.line_count().to_string(), + &resp.word_count().to_string(), &resp.content_length().to_string(), &resp.url().to_string(), ); diff --git a/src/utils.rs b/src/utils.rs index 572710b5..0279cd4a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -260,9 +260,9 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { let fancy_message = format!("{} !=> {}", url, last_redirect); let report = if let Some(msg_status) = e.status() { - create_report_string(msg_status.as_str(), "-1", &fancy_message) + create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message) } else { - create_report_string("UNK", "-1", &fancy_message) + create_report_string("UNK", "-1", "-1", "-1", &fancy_message) }; ferox_print(&report, &PROGRESS_PRINTER) @@ -278,15 +278,24 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { /// Helper to create the standard line for output to file/terminal /// /// example output: -/// 200 3280 https://localhost.com/FAQ -pub fn create_report_string(status: &str, content_length: &str, url: &str) -> String { +/// 200 127l 283w 4134c http://localhost/faq +pub fn create_report_string( + status: &str, + line_count: &str, + word_count: &str, + content_length: &str, + url: &str, +) -> String { if CONFIGURATION.quiet { // -q used, just need the url format!("{}\n", url) } else { - // normal printing with status and size + // normal printing with status and sizes let color_status = status_colorizer(status); - format!("{} {:>10} {}\n", color_status, content_length, url) + format!( + "{} {:>8}l {:>8}w {:>8}c {}\n", + color_status, line_count, word_count, content_length, url + ) } } From a93fe91459a03ca62140723a0f8a25cfbe14e3c0 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 17 Nov 2020 18:57:19 -0600 Subject: [PATCH 7/7] fixed a comment that didnt make sense --- src/client.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client.rs b/src/client.rs index eec7e336..bfa1b8d4 100644 --- a/src/client.rs +++ b/src/client.rs @@ -33,6 +33,7 @@ pub fn initialize( .redirect(policy); let client = match proxy { + // a proxy is specified, need to add it to the client Some(some_proxy) => { if !some_proxy.is_empty() { // it's not an empty string @@ -56,7 +57,7 @@ pub fn initialize( client // Some("") was used? } } - // a proxy is specified, just need to add it to the client + // no proxy specified None => client, };