Merge pull request #1123 from Yamato-Security/1119-category-filtering

added category filtering
Yamato-Security · Jul 4, 2023 · d71c2b9 · d71c2b9
2 parents b2f9d36 + 1936276
commit d71c2b9
Show file tree

Hide file tree

Showing 17 changed files with 274 additions and 7 deletions.
diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md
@@ -5,6 +5,7 @@
 **新機能:**
 
 - `csv-timeline`コマンドに`-R, --remove-duplicate-data`オプションを追加し、`%Details%`、`%AllFieldInfo%`、`%ExtraFieldInfo%`列の重複フィールドデータを`DUP`という文字列に変換し、ファイルサイズの削減を行う。(#1056) (@hitenkoku)
+- `csv-timeline`と`json-timeline`コマンドに`--include-category`と`--exclude-category`オプションを追加した。`include-category`は、指定された`category`フィールドのルールのみをロードする。`--exclude-category`は、指定された`category`フィールドを持つルールをロードしない。 (#1119) (@hitenkoku)
 
 **改善:**
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,7 @@
 **New Features:**
 
 - Added the `-R, --remove-duplicate-data` option in the `csv-timeline` command to replace duplicate field data with the string `DUP` in the `%Details%`, `%AllFieldInfo%`, `%ExtraFieldInfo%` columns to reduce file size. (#1056) (@hitenkoku)
+- Added `--include-category` and `--exclude-category` options to `csv-timeline` and `json-timeline` commands. When using `--include-category`, only rules with the specified `category` field will be loaded. `--exclude-category` will exclude rules from being loaded based on `category`. (#1119) (@hitenkoku)
 
 **Enhancements:**
 

diff --git a/src/afterfact.rs b/src/afterfact.rs
@@ -1809,6 +1809,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv.csv").to_path_buf()),
@@ -1885,6 +1887,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             };
             let ch = mock_ch_filter
                 .get(&CompactString::from("security"))
@@ -2108,6 +2112,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv_multiline.csv").to_path_buf()),
@@ -2186,6 +2192,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             };
             let ch = mock_ch_filter
                 .get(&CompactString::from("security"))
@@ -2395,6 +2403,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv_remove_duplicate.csv").to_path_buf()),
@@ -2471,6 +2481,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             };
             let ch = mock_ch_filter
                 .get(&CompactString::from("security"))
@@ -2702,6 +2714,8 @@ mod tests {
             enable_unsupported_rules: false,
             clobber: false,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         };
         let data: Vec<(CompactString, Profile)> = vec![
             (
@@ -2827,6 +2841,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv_json.json").to_path_buf()),
@@ -2901,6 +2917,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             };
             let ch = mock_ch_filter
                 .get(&CompactString::from("security"))
@@ -3061,6 +3079,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv_jsonl.jsonl").to_path_buf()),
@@ -3135,6 +3155,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             };
             let ch = mock_ch_filter
                 .get(&CompactString::from("security"))

diff --git a/src/detections/configs.rs b/src/detections/configs.rs
@@ -896,7 +896,7 @@ pub struct PivotKeywordOption {
     pub enable_unsupported_rules: bool,
 
     /// Ignore rules according to status (ex: experimental) (ex: stable,test)
-    #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 314)]
+    #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 315)]
     pub exclude_status: Option<Vec<String>>,
 
     /// Minimum level for rules (default: informational)
@@ -1014,13 +1014,21 @@ pub struct OutputOption {
     pub enable_unsupported_rules: bool,
 
     /// Ignore rules according to status (ex: experimental) (ex: stable,test)
-    #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 314)]
+    #[arg(help_heading = Some("Filtering"), long = "exclude-status", value_name = "STATUS", use_value_delimiter = true, value_delimiter = ',', display_order = 315)]
     pub exclude_status: Option<Vec<String>>,
 
     /// Only load rules with specific tags (ex: attack.execution,attack.discovery)
     #[arg(help_heading = Some("Filtering"), long = "tags", value_name = "TAGS", use_value_delimiter = true, value_delimiter = ',', display_order = 460)]
     pub tags: Option<Vec<String>>,
 
+    /// Only load rules with certain logsource categories (ex: process_creation,pipe_created)
+    #[arg(help_heading = Some("Filtering"), long = "include-category", value_name = "CATEGORY", conflicts_with = "exclude-category", use_value_delimiter = true, value_delimiter = ',', display_order = 351)]
+    pub include_category: Option<Vec<String>>,
+
+    /// Do not load rules with certain logsource categories (ex: process_creation,pipe_created)
+    #[arg(help_heading = Some("Filtering"), long = "exclude-category", value_name = "CATEGORY", conflicts_with = "include_category",use_value_delimiter = true, value_delimiter = ',', display_order = 314)]
+    pub exclude_category: Option<Vec<String>>,
+
     /// Minimum level for rules (default: informational)
     #[arg(
         help_heading = Some("Filtering"),
@@ -1544,6 +1552,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: option.enable_unsupported_rules,
             clobber: false,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         Action::Metrics(option) => Some(OutputOption {
             input_args: option.input_args.clone(),
@@ -1572,6 +1582,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: false,
             clobber: option.clobber,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         Action::LogonSummary(option) => Some(OutputOption {
             input_args: option.input_args.clone(),
@@ -1600,6 +1612,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: false,
             clobber: option.clobber,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         Action::Search(option) => Some(OutputOption {
             input_args: option.input_args.clone(),
@@ -1635,6 +1649,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: false,
             clobber: option.clobber,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         Action::SetDefaultProfile(option) => Some(OutputOption {
             input_args: InputOption {
@@ -1674,6 +1690,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: false,
             clobber: false,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         Action::UpdateRules(option) => Some(OutputOption {
             input_args: InputOption {
@@ -1713,6 +1731,8 @@ fn extract_output_options(config: &Config) -> Option<OutputOption> {
             enable_unsupported_rules: true,
             clobber: false,
             tags: None,
+            include_category: None,
+            exclude_category: None,
         }),
         _ => None,
     }

diff --git a/src/detections/detection.rs b/src/detections/detection.rs
@@ -1204,6 +1204,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,
@@ -1452,6 +1454,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()),
             output: Some(Path::new("./test_emit_csv.csv").to_path_buf()),
@@ -1573,6 +1577,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: Some(Path::new("test_files/mmdb").to_path_buf()),
             output: Some(Path::new("./test_emit_csv.csv").to_path_buf()),
@@ -1690,6 +1696,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv.csv").to_path_buf()),
@@ -1820,6 +1828,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("./test_emit_csv.csv").to_path_buf()),

diff --git a/src/detections/rule/condition_parser.rs b/src/detections/rule/condition_parser.rs
@@ -595,6 +595,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/detections/rule/count.rs b/src/detections/rule/count.rs
@@ -619,6 +619,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/detections/rule/matchers.rs b/src/detections/rule/matchers.rs
@@ -858,6 +858,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/detections/rule/mod.rs b/src/detections/rule/mod.rs
@@ -439,6 +439,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/detections/rule/selectionnodes.rs b/src/detections/rule/selectionnodes.rs
@@ -559,6 +559,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/detections/utils.rs b/src/detections/utils.rs
@@ -977,6 +977,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,

diff --git a/src/main.rs b/src/main.rs
@@ -1574,6 +1574,8 @@ mod tests {
                     enable_unsupported_rules: false,
                     clobber: false,
                     tags: None,
+                    include_category: None,
+                    exclude_category: None,
                 },
                 geo_ip: None,
                 output: None,
@@ -1723,6 +1725,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("overwrite.csv").to_path_buf()),
@@ -1793,6 +1797,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: true,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("overwrite.csv").to_path_buf()),
@@ -1861,6 +1867,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("overwrite.json").to_path_buf()),
@@ -1930,6 +1938,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: true,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: Some(Path::new("overwrite.json").to_path_buf()),

diff --git a/src/options/htmlreport.rs b/src/options/htmlreport.rs
@@ -285,6 +285,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: None,
@@ -336,6 +338,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             geo_ip: None,
             output: None,
@@ -390,6 +394,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             jsonl_timeline: false,
             geo_ip: None,
@@ -440,6 +446,8 @@ mod tests {
                 enable_unsupported_rules: false,
                 clobber: false,
                 tags: None,
+                include_category: None,
+                exclude_category: None,
             },
             jsonl_timeline: false,
             geo_ip: None,