From ab5fbeb6ede12595a3c39031fb70b52e51ac086c Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 8 Dec 2020 15:35:11 -0600 Subject: [PATCH 1/4] feature appears to be working; tests passing --- .gitignore | 3 + shell_completions/_feroxbuster | 2 +- src/config.rs | 173 +++++++++++++++++++++++---------- src/parser.rs | 2 +- tests/test_scanner.rs | 88 +++++++++-------- 5 files changed, 169 insertions(+), 99 deletions(-) diff --git a/.gitignore b/.gitignore index e4745d64..68ddcf84 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ img/** # scripts to check code coverage using nightly compiler check-coverage.sh lcov_cobertura.py + +# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is +.dockerignore diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 7b067fdc..d4a9ceb4 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -26,7 +26,7 @@ _feroxbuster() { '*--status-codes+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \ '-o+[Output file to write results to (use w/ --json for JSON entries)]' \ '--output+[Output file to write results to (use w/ --json for JSON entries)]' \ -'(-w --wordlist -u --url -t --threads -d --depth -T --timeout -v --verbosity -p --proxy -P --replay-proxy -R --replay-codes -s --status-codes -q --quiet --json -D --dont-filter -o --output --debug-log -a --user-agent -r --redirects -k --insecure -x --extensions -H --headers -Q --query -n --no-recursion -f --add-slash --stdin -S --filter-size -X --filter-regex -W --filter-words -N --filter-lines -C --filter-status -e --extract-links -L --scan-limit)--resume-from+[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \ +'(-u --url)--resume-from+[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \ '--debug-log+[Output file to write log entries (use w/ --json for JSON entries)]' \ '-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \ '--user-agent+[Sets the User-Agent (default: feroxbuster/VERSION)]' \ diff --git a/src/config.rs b/src/config.rs index f4dc029e..9fe8bf37 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,7 +2,7 @@ use crate::scan_manager::resume_scan; use crate::utils::{module_colorizer, status_colorizer}; use crate::{client, parser, progress}; use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION}; -use clap::value_t; +use clap::{value_t, ArgMatches}; use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget}; use lazy_static::lazy_static; use reqwest::{Client, StatusCode}; @@ -25,6 +25,32 @@ lazy_static! { pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false); } +/// macro helper to abstract away repetitive configuration updates +macro_rules! update_config_if_present { + ($c:expr, $m:ident, $v:expr, $t:ty) => { + match value_t!($m, $v, $t) { + Ok(value) => *$c = value, // Update value + Err(clap::Error { + kind: clap::ErrorKind::ArgumentNotFound, + message: _, + info: _, + }) => { + // Do nothing if argument not found + } + Err(e) => e.exit(), // Exit with error on parse error + } + }; +} + +/// macro helper to abstract away repetitive if not default: update checks +macro_rules! update_if_not_default { + ($old:expr, $new:expr, $default:expr) => { + if $new != $default { + *$old = $new; + } + }; +} + /// simple helper to clean up some code reuse below; panics under test / exits in prod fn report_and_exit(err: &str) -> ! { eprintln!( @@ -81,7 +107,7 @@ pub struct Configuration { pub status_codes: Vec, /// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code) - #[serde(default)] + #[serde(default = "status_codes")] pub replay_codes: Vec, /// Status Codes to filter out (deny list) @@ -367,11 +393,29 @@ impl Configuration { let args = parser::initialize().get_matches(); + // Get the default configuration, this is what will apply if nothing + // else is specified. + let mut config = Configuration::default(); + + // read in all config files + Self::parse_config_files(&mut config); + + // read in the user provided options, this produces a separate instance of Configuration + // in order to allow for potentially merging into a --resume-from Configuration + let cli_config = Self::parse_cli_args(&args); + + // --resume-from used, need to first read the Configuration from disk, and then + // merge the cli_config into the resumed config if let Some(filename) = args.value_of("resume_from") { // when resuming a scan, instead of normal configuration loading, we just // load the config from disk by calling resume_scan let mut previous_config = resume_scan(filename); + // if any other arguments were passed on the command line, the theory is that the + // user meant to modify the previously cancelled/saved scan in some way that we + // should take into account + Self::merge_config(&mut previous_config, cli_config); + // the resumed flag isn't printed in the banner and really has no business being // serialized or included in much of the usual config logic; simply setting it to true // here and being done with it @@ -388,10 +432,16 @@ impl Configuration { return previous_config; } - // Get the default configuration, this is what will apply if nothing - // else is specified. - let mut config = Configuration::default(); + // if we've gotten to this point in the code, --resume-from was not used, so we need to + // merge the cli options into the config file options and return the result + Self::merge_config(&mut config, cli_config); + + config + } + /// Parse all possible versions of the ferox-config.toml file, adhering to the order of + /// precedence outlined above + fn parse_config_files(mut config: &mut Self) { // Next, we parse the ferox-config.toml file, if present and set the values // therein to overwrite our default values. Deserialized defaults are specified // in the Configuration struct so that we don't change anything that isn't @@ -433,22 +483,12 @@ impl Configuration { let config_file = cwd.join(DEFAULT_CONFIG_NAME); Self::parse_and_merge_config(config_file, &mut config); } + } - macro_rules! update_config_if_present { - ($c:expr, $m:ident, $v:expr, $t:ty) => { - match value_t!($m, $v, $t) { - Ok(value) => *$c = value, // Update value - Err(clap::Error { - kind: clap::ErrorKind::ArgumentNotFound, - message: _, - info: _, - }) => { - // Do nothing if argument not found - } - Err(e) => e.exit(), // Exit with error on parse error - } - }; - } + /// Given a set of ArgMatches read from the CLI, update and return the default Configuration + /// settings + fn parse_cli_args(args: &ArgMatches) -> Self { + let mut config = Configuration::default(); update_config_if_present!(&mut config.threads, args, "threads", usize); update_config_if_present!(&mut config.depth, args, "depth", usize); @@ -562,8 +602,8 @@ impl Configuration { if args.is_present("stdin") { config.stdin = true; - } else { - config.target_url = String::from(args.value_of("url").unwrap()); + } else if let Some(url) = args.value_of("url") { + config.target_url = String::from(url); } //// @@ -681,38 +721,63 @@ impl Configuration { } /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge` - fn merge_config(settings: &mut Self, settings_to_merge: Self) { - settings.threads = settings_to_merge.threads; - settings.wordlist = settings_to_merge.wordlist; - settings.status_codes = settings_to_merge.status_codes; - settings.proxy = settings_to_merge.proxy; - settings.timeout = settings_to_merge.timeout; - settings.verbosity = settings_to_merge.verbosity; - settings.quiet = settings_to_merge.quiet; - settings.output = settings_to_merge.output; - settings.user_agent = settings_to_merge.user_agent; - settings.redirects = settings_to_merge.redirects; - settings.insecure = settings_to_merge.insecure; - settings.extract_links = settings_to_merge.extract_links; - settings.extensions = settings_to_merge.extensions; - settings.headers = settings_to_merge.headers; - settings.queries = settings_to_merge.queries; - settings.no_recursion = settings_to_merge.no_recursion; - settings.add_slash = settings_to_merge.add_slash; - settings.stdin = settings_to_merge.stdin; - settings.depth = settings_to_merge.depth; - settings.filter_size = settings_to_merge.filter_size; - settings.filter_regex = settings_to_merge.filter_regex; - settings.filter_word_count = settings_to_merge.filter_word_count; - settings.filter_line_count = settings_to_merge.filter_line_count; - settings.filter_status = settings_to_merge.filter_status; - settings.dont_filter = settings_to_merge.dont_filter; - settings.scan_limit = settings_to_merge.scan_limit; - settings.replay_proxy = settings_to_merge.replay_proxy; - settings.replay_codes = settings_to_merge.replay_codes; - settings.save_state = settings_to_merge.save_state; - settings.debug_log = settings_to_merge.debug_log; - settings.json = settings_to_merge.json; + fn merge_config(conf: &mut Self, new: Self) { + // does not include the following Configuration fields, as they don't make sense here + // - kind + // - client + // - replay_client + // - resumed + // - config + update_if_not_default!(&mut conf.target_url, new.target_url, ""); + update_if_not_default!(&mut conf.proxy, new.proxy, ""); + update_if_not_default!(&mut conf.verbosity, new.verbosity, 0); + update_if_not_default!(&mut conf.quiet, new.quiet, false); + update_if_not_default!(&mut conf.output, new.output, ""); + update_if_not_default!(&mut conf.redirects, new.redirects, false); + update_if_not_default!(&mut conf.insecure, new.insecure, false); + update_if_not_default!(&mut conf.extract_links, new.extract_links, false); + update_if_not_default!(&mut conf.extensions, new.extensions, Vec::::new()); + update_if_not_default!(&mut conf.headers, new.headers, HashMap::new()); + update_if_not_default!(&mut conf.queries, new.queries, Vec::new()); + update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false); + update_if_not_default!(&mut conf.add_slash, new.add_slash, false); + update_if_not_default!(&mut conf.stdin, new.stdin, false); + update_if_not_default!(&mut conf.filter_size, new.filter_size, Vec::::new()); + update_if_not_default!( + &mut conf.filter_regex, + new.filter_regex, + Vec::::new() + ); + update_if_not_default!( + &mut conf.filter_word_count, + new.filter_word_count, + Vec::::new() + ); + update_if_not_default!( + &mut conf.filter_line_count, + new.filter_line_count, + Vec::::new() + ); + update_if_not_default!( + &mut conf.filter_status, + new.filter_status, + Vec::::new() + ); + update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false); + update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0); + update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, ""); + update_if_not_default!(&mut conf.debug_log, new.debug_log, ""); + update_if_not_default!(&mut conf.json, new.json, false); + + update_if_not_default!(&mut conf.timeout, new.timeout, timeout()); + update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent()); + update_if_not_default!(&mut conf.threads, new.threads, threads()); + update_if_not_default!(&mut conf.depth, new.depth, depth()); + update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist()); + update_if_not_default!(&mut conf.status_codes, new.status_codes, status_codes()); + // status_codes() is the default for replay_codes, if they're not provided + update_if_not_default!(&mut conf.replay_codes, new.replay_codes, status_codes()); + update_if_not_default!(&mut conf.save_state, new.save_state, save_state()); } /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values diff --git a/src/parser.rs b/src/parser.rs index 31eecf46..76c5c7f9 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -135,7 +135,7 @@ pub fn initialize() -> App<'static, 'static> { .long("resume-from") .value_name("STATE_FILE") .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)") - .conflicts_with_all(&["wordlist", "url", "threads", "depth", "timeout", "verbosity", "proxy", "replay_proxy", "replay_codes", "status_codes", "quiet", "json", "dont_filter", "output", "debug_log", "user_agent", "redirects", "insecure", "extensions", "headers", "queries", "no_recursion", "add_slash", "stdin", "filter_size", "filter_regex", "filter_words", "filter_lines", "filter_status", "extract_links", "scan_limit"]) + .conflicts_with("url") .takes_value(true), ) .arg( diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index b9a121cf..d53d0a38 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -365,25 +365,26 @@ fn scanner_single_request_returns_301_without_location_header( } #[test] -/// send a single valid request, filter the size of the response, expect one out of 2 urls -fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box> { +/// send a single valid request, expect a 200 response that then gets routed to the replay +/// proxy +fn scanner_single_request_replayed_to_proxy() -> Result<(), Box> { let srv = MockServer::start(); - let (tmp_dir, file) = - setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?; + let proxy = MockServer::start(); + let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; let mock = Mock::new() .expect_method(GET) .expect_path("/LICENSE") .return_status(200) - .return_body("this is a not a test") + .return_body("this is a test") .create_on(&srv); - let filtered_mock = Mock::new() + let mock_two = Mock::new() .expect_method(GET) - .expect_path("/ignored") + .expect_path("/LICENSE") .return_status(200) .return_body("this is a test") - .create_on(&srv); + .create_on(&proxy); let cmd = Command::cargo_bin("feroxbuster") .unwrap() @@ -391,48 +392,49 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box Result<(), Box> { +/// send a single valid request, filter the size of the response, expect one out of 2 urls +fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box> { let srv = MockServer::start(); - let proxy = MockServer::start(); - let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; + let (tmp_dir, file) = + setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?; let mock = Mock::new() .expect_method(GET) .expect_path("/LICENSE") .return_status(200) - .return_body("this is a test") + .return_body("this is a not a test") .create_on(&srv); - let mock_two = Mock::new() + let filtered_mock = Mock::new() .expect_method(GET) - .expect_path("/LICENSE") + .expect_path("/ignored") .return_status(200) .return_body("this is a test") - .create_on(&proxy); + .create_on(&srv); let cmd = Command::cargo_bin("feroxbuster") .unwrap() @@ -440,23 +442,23 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box Date: Tue, 8 Dec 2020 20:37:07 -0600 Subject: [PATCH 2/4] fixed replay proxy issue --- Cargo.toml | 2 +- src/config.rs | 5 +++-- tests/test_scanner.rs | 2 -- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 229ed28b..851810b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.9.0" +version = "1.9.1" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" diff --git a/src/config.rs b/src/config.rs index 9fe8bf37..b1740276 100644 --- a/src/config.rs +++ b/src/config.rs @@ -436,6 +436,9 @@ impl Configuration { // merge the cli options into the config file options and return the result Self::merge_config(&mut config, cli_config); + // rebuild clients is the last step in either code branch + Self::try_rebuild_clients(&mut config); + config } @@ -649,8 +652,6 @@ impl Configuration { } } - Self::try_rebuild_clients(&mut config); - config } diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index d53d0a38..8155366e 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -396,8 +396,6 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box Date: Tue, 8 Dec 2020 20:49:38 -0600 Subject: [PATCH 3/4] fixed test with hardcoded ferox version in expected msg --- src/scan_manager.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 3e3fcfe1..f40f3433 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -698,6 +698,7 @@ pub fn resume_scan(filename: &str) -> Configuration { #[cfg(test)] mod tests { use super::*; + use crate::VERSION; use predicates::prelude::*; #[test] @@ -983,8 +984,8 @@ mod tests { let json_state = ferox_state.as_json(); let expected = format!( - r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#, - saved_id + r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#, + saved_id, VERSION ); assert!(predicates::str::similar(expected).eval(&json_state)); From 3caa8d2cebc759d7bdec9ce1b430d0beeced4f45 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 9 Dec 2020 07:52:33 -0600 Subject: [PATCH 4/4] comment lint and corrections --- src/main.rs | 4 ++-- src/scan_manager.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index cec6aba9..cf9b1ffe 100644 --- a/src/main.rs +++ b/src/main.rs @@ -180,8 +180,8 @@ async fn get_targets() -> FeroxResult> { targets.push(line?); } } else if CONFIGURATION.resumed { - // resume-from can't be used with any other flag, making it mutually exclusive from either - // of the other two options + // resume-from can't be used with --url, and --stdin is marked false for every resumed + // scan, making it mutually exclusive from either of the other two options if let Ok(scans) = SCANNED_URLS.scans.lock() { for scan in scans.iter() { // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from diff --git a/src/scan_manager.rs b/src/scan_manager.rs index f40f3433..d7059f3f 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -668,7 +668,6 @@ pub fn resume_scan(filename: &str) -> Configuration { std::process::exit(1); }); - // let scans: FeroxScans = serde_json::from_value(state.get("scans").unwrap().clone()).unwrap(); if let Some(responses) = state.get("responses") { if let Some(arr_responses) = responses.as_array() { for response in arr_responses {