From 18880623ef17fa2bc856e866f0a29e7d471b0c6e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 00:49:44 +0900 Subject: [PATCH 1/7] WIP(config): added include-category and exclude-category option and reorder option #1119 --- src/detections/configs.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 9c3aaf2f6..2100b985e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -896,7 +896,7 @@ pub struct PivotKeywordOption { pub enable_unsupported_rules: bool, /// Ignore rules according to status (ex: experimental) (ex: stable,test) - #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 314)] + #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 315)] pub exclude_status: Option>, /// Minimum level for rules (default: informational) @@ -1014,13 +1014,21 @@ pub struct OutputOption { pub enable_unsupported_rules: bool, /// Ignore rules according to status (ex: experimental) (ex: stable,test) - #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 314)] + #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 315)] pub exclude_status: Option>, /// Only load rules with specific tags (ex: attack.execution,attack.discovery) #[arg(help_heading = Some("Filtering"), long = "tags", value_name = "TAGS", use_value_delimiter = true, value_delimiter = ',', display_order = 460)] pub tags: Option>, + /// Only load rules with certain logsource categories (ex: process_creation,pipe_created) + #[arg(help_heading = Some("Filtering"), long = "include-category", value_name = "CATEGORY", conflicts_with = "exclude-category", use_value_delimiter = true, value_delimiter = ',', display_order = 351)] + pub include_category: Option>, + + /// Do not load rules with certain logsource categories (ex: process_creation,pipe_created) + #[arg(help_heading = Some("Filtering"), long = "exclude-category", value_name = "CATEGORY", conflicts_with = "include_category",use_value_delimiter = true, value_delimiter = ',', display_order = 314)] + pub exclude_category: Option>, + /// Minimum level for rules (default: informational) #[arg( help_heading = Some("Filtering"), @@ -1544,6 +1552,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: option.enable_unsupported_rules, clobber: false, tags: None, + include_category: None, + exclude_category: None, }), Action::Metrics(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -1572,6 +1582,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: false, clobber: option.clobber, tags: None, + include_category: None, + exclude_category: None, }), Action::LogonSummary(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -1600,6 +1612,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: false, clobber: option.clobber, tags: None, + include_category: None, + exclude_category: None, }), Action::Search(option) => Some(OutputOption { input_args: option.input_args.clone(), @@ -1635,6 +1649,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: false, clobber: option.clobber, tags: None, + include_category: None, + exclude_category: None, }), Action::SetDefaultProfile(option) => Some(OutputOption { input_args: InputOption { @@ -1674,6 +1690,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }), Action::UpdateRules(option) => Some(OutputOption { input_args: InputOption { @@ -1713,6 +1731,8 @@ fn extract_output_options(config: &Config) -> Option { enable_unsupported_rules: true, clobber: false, tags: None, + include_category: None, + exclude_category: None, }), _ => None, } From 9ed350a7eaa34506da5881beb984a47a3f93b83c Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 01:35:16 +0900 Subject: [PATCH 2/7] feat(yaml): added include-category and exclude-category processing #1119 --- src/yaml.rs | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/yaml.rs b/src/yaml.rs index c1b63660f..ed3722a2c 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -240,7 +240,7 @@ impl ParseYaml { io::Result::Ok(ret) })?; } - + let exist_output_opt = stored_static.output_option.is_some(); let files = yaml_docs.into_iter().filter_map(|(filepath, yaml_doc)| { //除外されたルールは無視する let rule_id = &yaml_doc["id"].as_str(); @@ -284,7 +284,7 @@ impl ParseYaml { *entry += 1; return Option::None; } - if stored_static.output_option.is_some() + if exist_output_opt && ((s == "deprecated" && !stored_static .output_option @@ -304,9 +304,33 @@ impl ParseYaml { } } + if exist_output_opt { + let category_in_rule = yaml_doc["logsource"]["category"].as_str().unwrap_or_default(); + let mut include_category = &Vec::default(); + let mut exclude_category = &Vec::default(); + + if let Some(tmp) = &stored_static.output_option.as_ref().unwrap().include_category { + include_category = tmp; + } + + if let Some(tmp) = &stored_static.output_option.as_ref().unwrap().exclude_category { + exclude_category = tmp; + } + + if !include_category.is_empty() && !include_category.contains(&category_in_rule.to_string()) { + let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); + *entry += 1; + return Option::None; + } + if !exclude_category.is_empty() && exclude_category.contains(&category_in_rule.to_string()) { + let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); + *entry += 1; + return Option::None; + } + } + // tags optionで指定されたtagsを持たないルールは除外する - if stored_static.output_option.is_some() - && stored_static.output_option.as_ref().unwrap().tags.is_some() + if exist_output_opt && stored_static.output_option.as_ref().unwrap().tags.is_some() { let target_tags = stored_static .output_option From f8ab22526928e2367ef1bc5a2617bc1ff5220760 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 01:37:12 +0900 Subject: [PATCH 3/7] test: fixed test due to adding include-category and exclude-category options #1119 --- src/afterfact.rs | 22 ++++++++++++++++++++++ src/detections/detection.rs | 10 ++++++++++ src/detections/rule/condition_parser.rs | 2 ++ src/detections/rule/count.rs | 2 ++ src/detections/rule/matchers.rs | 2 ++ src/detections/rule/mod.rs | 2 ++ src/detections/rule/selectionnodes.rs | 2 ++ src/detections/utils.rs | 2 ++ src/main.rs | 10 ++++++++++ src/options/htmlreport.rs | 8 ++++++++ src/options/profile.rs | 8 ++++++++ src/yaml.rs | 2 ++ 12 files changed, 72 insertions(+) diff --git a/src/afterfact.rs b/src/afterfact.rs index 11d0098a9..7c45807a7 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -1809,6 +1809,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1885,6 +1887,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2108,6 +2112,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_multiline.csv").to_path_buf()), @@ -2186,6 +2192,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2395,6 +2403,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_remove_duplicate.csv").to_path_buf()), @@ -2471,6 +2481,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -2702,6 +2714,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let data: Vec<(CompactString, Profile)> = vec![ ( @@ -2827,6 +2841,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_json.json").to_path_buf()), @@ -2901,6 +2917,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let ch = mock_ch_filter .get(&CompactString::from("security")) @@ -3061,6 +3079,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv_jsonl.jsonl").to_path_buf()), @@ -3135,6 +3155,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }; let ch = mock_ch_filter .get(&CompactString::from("security")) diff --git a/src/detections/detection.rs b/src/detections/detection.rs index fa84fe587..d29e3a855 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1204,6 +1204,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -1452,6 +1454,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1573,6 +1577,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()), output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1690,6 +1696,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), @@ -1820,6 +1828,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("./test_emit_csv.csv").to_path_buf()), diff --git a/src/detections/rule/condition_parser.rs b/src/detections/rule/condition_parser.rs index 27a704fd7..7ef6a8fad 100644 --- a/src/detections/rule/condition_parser.rs +++ b/src/detections/rule/condition_parser.rs @@ -595,6 +595,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/detections/rule/count.rs b/src/detections/rule/count.rs index 42f1c4ee0..9f494d08e 100644 --- a/src/detections/rule/count.rs +++ b/src/detections/rule/count.rs @@ -619,6 +619,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/detections/rule/matchers.rs b/src/detections/rule/matchers.rs index 51e3ea3ac..09bf51def 100644 --- a/src/detections/rule/matchers.rs +++ b/src/detections/rule/matchers.rs @@ -858,6 +858,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/detections/rule/mod.rs b/src/detections/rule/mod.rs index 5f87aa3bd..381ed02dd 100644 --- a/src/detections/rule/mod.rs +++ b/src/detections/rule/mod.rs @@ -439,6 +439,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/detections/rule/selectionnodes.rs b/src/detections/rule/selectionnodes.rs index ded010c1f..84068abee 100644 --- a/src/detections/rule/selectionnodes.rs +++ b/src/detections/rule/selectionnodes.rs @@ -559,6 +559,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/detections/utils.rs b/src/detections/utils.rs index db2dc9cef..7bb5a0171 100644 --- a/src/detections/utils.rs +++ b/src/detections/utils.rs @@ -977,6 +977,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/main.rs b/src/main.rs index 0fe6ebc58..ae624afdc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1574,6 +1574,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -1723,6 +1725,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -1793,6 +1797,8 @@ mod tests { enable_unsupported_rules: false, clobber: true, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("overwrite.csv").to_path_buf()), @@ -1861,6 +1867,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), @@ -1930,6 +1938,8 @@ mod tests { enable_unsupported_rules: false, clobber: true, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: Some(Path::new("overwrite.json").to_path_buf()), diff --git a/src/options/htmlreport.rs b/src/options/htmlreport.rs index 165bb2aa8..892452fdc 100644 --- a/src/options/htmlreport.rs +++ b/src/options/htmlreport.rs @@ -285,6 +285,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -336,6 +338,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -390,6 +394,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, jsonl_timeline: false, geo_ip: None, @@ -440,6 +446,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, jsonl_timeline: false, geo_ip: None, diff --git a/src/options/profile.rs b/src/options/profile.rs index 8b754048c..afbb4c09e 100644 --- a/src/options/profile.rs +++ b/src/options/profile.rs @@ -447,6 +447,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -559,6 +561,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -621,6 +625,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, @@ -713,6 +719,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, diff --git a/src/yaml.rs b/src/yaml.rs index ed3722a2c..423f5edc4 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -453,6 +453,8 @@ mod tests { enable_unsupported_rules: false, clobber: false, tags: None, + include_category: None, + exclude_category: None, }, geo_ip: None, output: None, From 6cd300554f6288277946beab20cf26f0236747d2 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 01:38:04 +0900 Subject: [PATCH 4/7] test(yaml): added test for include-category and exclude-category #1119 --- src/yaml.rs | 109 +++++++++++++++++++++++ test_files/rules/level_yaml/critical.yml | 1 + test_files/rules/level_yaml/high.yml | 1 + 3 files changed, 111 insertions(+) diff --git a/src/yaml.rs b/src/yaml.rs index 423f5edc4..cf3903458 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -780,4 +780,113 @@ mod tests { .unwrap(); assert_eq!(yaml.files.len(), 3); } + + #[test] + fn test_include_category_option_1opt() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().include_category = + Some(vec!["test_category1".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 1); + } + + #[test] + fn test_include_category_option_multi_opt() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().include_category = + Some(vec!["test_category1".to_string(), "test_category2".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 2); + } + + #[test] + fn test_include_category_option_not_found() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().include_category = + Some(vec!["not found".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 0); + } + + #[test] + fn test_exclude_category_option_1opt() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().exclude_category = + Some(vec!["test_category1".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 4); + } + + #[test] + fn test_exclude_category_option_multi_opt() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().exclude_category = + Some(vec!["test_category1".to_string(), "test_category2".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 3); + } + + #[test] + fn test_exclude_category_option_notfound() { + let path = Path::new("test_files/rules/level_yaml"); + let mut dummy_stored_static = create_dummy_stored_static(); + dummy_stored_static.output_option.as_mut().unwrap().exclude_category = + Some(vec!["not found".to_string()]); + let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); + yaml.read_dir( + path, + "", + "", + &filter::exclude_ids(&dummy_stored_static), + &dummy_stored_static, + ) + .unwrap(); + assert_eq!(yaml.files.len(), 5); + } + } diff --git a/test_files/rules/level_yaml/critical.yml b/test_files/rules/level_yaml/critical.yml index cda333512..8a492a51f 100644 --- a/test_files/rules/level_yaml/critical.yml +++ b/test_files/rules/level_yaml/critical.yml @@ -3,6 +3,7 @@ description: hogehoge enabled: true author: Yea logsource: + category: test_category1 product: windows detection: selection: diff --git a/test_files/rules/level_yaml/high.yml b/test_files/rules/level_yaml/high.yml index e3c7e89be..e6c116ac8 100644 --- a/test_files/rules/level_yaml/high.yml +++ b/test_files/rules/level_yaml/high.yml @@ -3,6 +3,7 @@ description: hogehoge enabled: true author: Yea logsource: + category: test_category2 product: windows detection: selection: From 214f949ca8b19b3016a5dd65e73c968c536666de Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 01:45:03 +0900 Subject: [PATCH 5/7] docs(CHANGELOG): added #1119 --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 1f7fd19f8..060037a14 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -11,6 +11,7 @@ - 新しいログ形式の`.evtx`を使用するWindows Vistaがリリースされた2007年1月31日以前のタイムスタンプを持つ破損されたイベントレコードを無視するようにした。(#1102) (@fukusuket) - `metrics`コマンドで`--output`オプションを指定した時に標準出力に結果を表示しないように変更した。 (#1099) (@hitenkoku) - `csv-timeline` コマンドと `json-timeline` コマンドに `--tags` オプションを追加し、指定した `tags` フィールドを持つルールのみでスキャンできるようにした。() +- csv-timeline` と `json-timeline` コマンドに `--include-category` と `--exclude-category` オプションを追加した。include-category` は指定された `category` フィールドでのみスキャンを許可する。cv-timeline` と `json-timeline` コマンドのオプション `--exclude-category` は指定した `category` フィールドを持つルールのスキャンを除外する。 (#1119) (@hitenkoku) **バグ修正:** diff --git a/CHANGELOG.md b/CHANGELOG.md index c7b5e2463..6afbf8397 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Ignore corrupted event records with timestamps before 2007/1/31 when Windows Vista was released with the new `.evtx` log format. (#1102) (@fukusuket) - Modified don't output to terminal when `--output` is set in `metrics` command. (#1099) (@hitenkoku) - Added `--tags` option to `csv-timeline` and `json-timeline` commands to allow scanning only for rules with the specified `tags` field. (#1108) (@hitenkoku) +- Added `--include-category` and `--exclude-category` option to `csv-timeline` and `json-timeline` commands. `--include-category` allow scanning only with the specified `category` field. `--exclude-category` exclude scanning with rule has the field . (#1119) (@hitenkoku) **Bug Fixes:** From 1c8bed215cd310f0853d327d266bf39f3cab3074 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 4 Jul 2023 01:45:26 +0900 Subject: [PATCH 6/7] style: cargo fmt --- src/yaml.rs | 78 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 20 deletions(-) diff --git a/src/yaml.rs b/src/yaml.rs index cf3903458..99f04cd47 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -305,24 +305,40 @@ impl ParseYaml { } if exist_output_opt { - let category_in_rule = yaml_doc["logsource"]["category"].as_str().unwrap_or_default(); + let category_in_rule = yaml_doc["logsource"]["category"] + .as_str() + .unwrap_or_default(); let mut include_category = &Vec::default(); let mut exclude_category = &Vec::default(); - if let Some(tmp) = &stored_static.output_option.as_ref().unwrap().include_category { + if let Some(tmp) = &stored_static + .output_option + .as_ref() + .unwrap() + .include_category + { include_category = tmp; } - if let Some(tmp) = &stored_static.output_option.as_ref().unwrap().exclude_category { + if let Some(tmp) = &stored_static + .output_option + .as_ref() + .unwrap() + .exclude_category + { exclude_category = tmp; } - if !include_category.is_empty() && !include_category.contains(&category_in_rule.to_string()) { + if !include_category.is_empty() + && !include_category.contains(&category_in_rule.to_string()) + { let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); *entry += 1; return Option::None; } - if !exclude_category.is_empty() && exclude_category.contains(&category_in_rule.to_string()) { + if !exclude_category.is_empty() + && exclude_category.contains(&category_in_rule.to_string()) + { let entry = self.rule_load_cnt.entry("excluded".into()).or_insert(0); *entry += 1; return Option::None; @@ -330,8 +346,7 @@ impl ParseYaml { } // tags optionで指定されたtagsを持たないルールは除外する - if exist_output_opt && stored_static.output_option.as_ref().unwrap().tags.is_some() - { + if exist_output_opt && stored_static.output_option.as_ref().unwrap().tags.is_some() { let target_tags = stored_static .output_option .as_ref() @@ -785,8 +800,11 @@ mod tests { fn test_include_category_option_1opt() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().include_category = - Some(vec!["test_category1".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .include_category = Some(vec!["test_category1".to_string()]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -803,8 +821,14 @@ mod tests { fn test_include_category_option_multi_opt() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().include_category = - Some(vec!["test_category1".to_string(), "test_category2".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .include_category = Some(vec![ + "test_category1".to_string(), + "test_category2".to_string(), + ]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -821,8 +845,11 @@ mod tests { fn test_include_category_option_not_found() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().include_category = - Some(vec!["not found".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .include_category = Some(vec!["not found".to_string()]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -839,8 +866,11 @@ mod tests { fn test_exclude_category_option_1opt() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().exclude_category = - Some(vec!["test_category1".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .exclude_category = Some(vec!["test_category1".to_string()]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -857,8 +887,14 @@ mod tests { fn test_exclude_category_option_multi_opt() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().exclude_category = - Some(vec!["test_category1".to_string(), "test_category2".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .exclude_category = Some(vec![ + "test_category1".to_string(), + "test_category2".to_string(), + ]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -875,8 +911,11 @@ mod tests { fn test_exclude_category_option_notfound() { let path = Path::new("test_files/rules/level_yaml"); let mut dummy_stored_static = create_dummy_stored_static(); - dummy_stored_static.output_option.as_mut().unwrap().exclude_category = - Some(vec!["not found".to_string()]); + dummy_stored_static + .output_option + .as_mut() + .unwrap() + .exclude_category = Some(vec!["not found".to_string()]); let mut yaml = yaml::ParseYaml::new(&dummy_stored_static); yaml.read_dir( path, @@ -888,5 +927,4 @@ mod tests { .unwrap(); assert_eq!(yaml.files.len(), 5); } - } From 19362768edaf9a90de0af77f22e3a1d9a99712a5 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Tue, 4 Jul 2023 08:40:32 +0900 Subject: [PATCH 7/7] update changelog --- CHANGELOG-Japanese.md | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 060037a14..de076b377 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -5,13 +5,13 @@ **新機能:** - `csv-timeline`コマンドに`-R, --remove-duplicate-data`オプションを追加し、`%Details%`、`%AllFieldInfo%`、`%ExtraFieldInfo%`列の重複フィールドデータを`DUP`という文字列に変換し、ファイルサイズの削減を行う。(#1056) (@hitenkoku) +- `csv-timeline`と`json-timeline`コマンドに`--include-category`と`--exclude-category`オプションを追加した。`include-category`は、指定された`category`フィールドのルールのみをロードする。`--exclude-category`は、指定された`category`フィールドを持つルールをロードしない。 (#1119) (@hitenkoku) **改善:** - 新しいログ形式の`.evtx`を使用するWindows Vistaがリリースされた2007年1月31日以前のタイムスタンプを持つ破損されたイベントレコードを無視するようにした。(#1102) (@fukusuket) - `metrics`コマンドで`--output`オプションを指定した時に標準出力に結果を表示しないように変更した。 (#1099) (@hitenkoku) - `csv-timeline` コマンドと `json-timeline` コマンドに `--tags` オプションを追加し、指定した `tags` フィールドを持つルールのみでスキャンできるようにした。() -- csv-timeline` と `json-timeline` コマンドに `--include-category` と `--exclude-category` オプションを追加した。include-category` は指定された `category` フィールドでのみスキャンを許可する。cv-timeline` と `json-timeline` コマンドのオプション `--exclude-category` は指定した `category` フィールドを持つルールのスキャンを除外する。 (#1119) (@hitenkoku) **バグ修正:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 6afbf8397..14da7d105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,13 @@ **New Features:** - Added the `-R, --remove-duplicate-data` option in the `csv-timeline` command to replace duplicate field data with the string `DUP` in the `%Details%`, `%AllFieldInfo%`, `%ExtraFieldInfo%` columns to reduce file size. (#1056) (@hitenkoku) +- Added `--include-category` and `--exclude-category` options to `csv-timeline` and `json-timeline` commands. When using `--include-category`, only rules with the specified `category` field will be loaded. `--exclude-category` will exclude rules from being loaded based on `category`. (#1119) (@hitenkoku) **Enhancements:** - Ignore corrupted event records with timestamps before 2007/1/31 when Windows Vista was released with the new `.evtx` log format. (#1102) (@fukusuket) - Modified don't output to terminal when `--output` is set in `metrics` command. (#1099) (@hitenkoku) - Added `--tags` option to `csv-timeline` and `json-timeline` commands to allow scanning only for rules with the specified `tags` field. (#1108) (@hitenkoku) -- Added `--include-category` and `--exclude-category` option to `csv-timeline` and `json-timeline` commands. `--include-category` allow scanning only with the specified `category` field. `--exclude-category` exclude scanning with rule has the field . (#1119) (@hitenkoku) **Bug Fixes:**