From b4f63ea74a3fcf0c34895cf1d0c9d424d6bc3b15 Mon Sep 17 00:00:00 2001 From: Hilko Bengen Date: Thu, 16 May 2024 00:21:46 +0200 Subject: [PATCH] Make filtering first event per process optional The first observed event in a process is special because it is used to uniquely identify that process when enriching PIDs. It is therefore useful to keep this first event in the main log. --- etc/laurel/config.toml | 6 ++ man/laurel.8.md | 4 + src/coalesce.rs | 116 +++++++++++++++++++----- src/config.rs | 3 + src/testdata/record-syscall-key.txt | 7 +- src/testdata/record-syscall-nullkey.txt | 10 +- 6 files changed, 116 insertions(+), 30 deletions(-) diff --git a/etc/laurel/config.toml b/etc/laurel/config.toml index b0eae32..2217c43 100644 --- a/etc/laurel/config.toml +++ b/etc/laurel/config.toml @@ -158,6 +158,12 @@ filter-null-keys = false # "^type=PATH msg=\\S*? item=\\S*? name=\"/var/run/nscd[.]sock\" " # ] +# Keep the first event observed for any given process even if it would +# be filtered otherwise. This should only be turned off if +# reproducible process tracking or process tree reconstruction is not +# required. +# keep-first-per-processes = true + # What to do with filtered events? "drop" or "log" to the filterlog # defined above. filter-action = "drop" diff --git a/man/laurel.8.md b/man/laurel.8.md index 0e0e815..352ea20 100644 --- a/man/laurel.8.md +++ b/man/laurel.8.md @@ -175,6 +175,10 @@ using them for internal processing such as process tracking. that contain such lines are then filtered. Default: empty - `filter-action`: What to do with filtered events? `drop` or `log` to the filterlog defined above. +- `keep-first-per-process`: Keep the first event observed for any + given process even if it would be filtered otherwise. This should + only be turned off if reproducible process tracking or process tree + reconstruction is not required. Default: true # SIGNALS diff --git a/src/coalesce.rs b/src/coalesce.rs index e4b40dc..4298fa7 100644 --- a/src/coalesce.rs +++ b/src/coalesce.rs @@ -49,6 +49,7 @@ pub struct Settings { pub filter_labels: HashSet>, pub filter_null_keys: bool, pub filter_raw_lines: regex::bytes::RegexSet, + pub filter_first_per_process: bool, } impl Default for Settings { @@ -75,6 +76,7 @@ impl Default for Settings { filter_labels: HashSet::new(), filter_null_keys: false, filter_raw_lines: regex::bytes::RegexSet::empty(), + filter_first_per_process: false, } } } @@ -656,6 +658,7 @@ impl<'a, 'ev> Coalesce<'a, 'ev> { let mut syscall_name: Option<&'static str> = None; let mut syscall_is_exec = false; + let mut force_keep = false; let mut current_process: Option = None; let mut parent: Option = None; @@ -747,11 +750,12 @@ impl<'a, 'ev> Coalesce<'a, 'ev> { // Process entry. syscall_is_exec = sn.contains("execve"); parent = self.processes.get_or_retrieve(proc.ppid).cloned(); - let pr = if !syscall_is_exec { - self.processes.get_or_retrieve(proc.pid) + let pr; + if syscall_is_exec { + pr = None; } else { - None - }; + pr = self.processes.get_or_retrieve(proc.pid); + } match pr { Some(pr) if proc.ppid == pr.ppid && proc.exe == pr.exe => { // existing, plausible process in table @@ -768,6 +772,10 @@ impl<'a, 'ev> Coalesce<'a, 'ev> { } _ => { // first syscall in new process + if !self.settings.filter_first_per_process { + ev.filter = false; + force_keep = true; + } proc.key = ProcessKey::Event(ev.id); if let Some(pa) = &parent { proc.parent = Some(pa.key); @@ -805,13 +813,13 @@ impl<'a, 'ev> Coalesce<'a, 'ev> { } if let Some(key) = &key { - if self.settings.filter_keys.contains(key.as_ref()) { + if !force_keep && self.settings.filter_keys.contains(key.as_ref()) { ev.filter = true; } if self.settings.proc_label_keys.contains(key.as_ref()) { proc.labels.insert(key.to_vec()); } - } else if self.settings.filter_null_keys { + } else if !force_keep && self.settings.filter_null_keys { ev.filter = true; } @@ -866,10 +874,11 @@ impl<'a, 'ev> Coalesce<'a, 'ev> { // filter early on labels if let Some(proc) = ¤t_process { self.processes.set_labels(&proc.key, &proc.labels); - if proc - .labels - .iter() - .any(|x| self.settings.filter_labels.contains(x)) + if !force_keep + && proc + .labels + .iter() + .any(|x| self.settings.filter_labels.contains(x)) { ev.filter = true; } @@ -1398,7 +1407,19 @@ mod test { c.settings.filter_keys.insert(Vec::from(&b"this-too"[..])); process_record(&mut c, include_bytes!("testdata/record-syscall-key.txt"))?; drop(c); - assert!(events.borrow().is_empty()); + // fist event for process -> don't filter + assert!(events + .borrow() + .iter() + .any(|e| &e.id == "1628602815.266:2365")); + assert!(!events + .borrow() + .iter() + .any(|e| &e.id == "1628602815.266:2366")); + assert!(!events + .borrow() + .iter() + .any(|e| &e.id == "1628602815.266:2367")); let mut c = Coalesce::new(mk_emit_vec(&events)); c.settings.filter_null_keys = true; @@ -1407,7 +1428,17 @@ mod test { include_bytes!("testdata/record-syscall-nullkey.txt"), )?; drop(c); - assert!(events.borrow().is_empty()); + + // not first event for process -> filter + assert!(!events + .borrow() + .iter() + .any(|e| &e.id == "1678282381.452:102337")); + // fist event for process -> don't filter + assert!(events + .borrow() + .iter() + .any(|e| &e.id == "1678283440.683:225")); let mut c = Coalesce::new(mk_emit_vec(&events)); c.settings @@ -1425,6 +1456,7 @@ mod test { let ec: Rc>> = Rc::new(RefCell::new(None)); let mut c = Coalesce::new(mk_emit(&ec)); + c.settings.filter_first_per_process = true; c.settings .proc_label_keys .insert(Vec::from(&b"software_mgmt"[..])); @@ -1464,11 +1496,15 @@ mod test { "^type=SOCKADDR (?:node=\\$*? )?msg=audit\\(\\S*?\\): saddr=01002F7661722F72756E2F6E7363642F736F636B657400", ]) .expect("failed to compile regex"); + c.settings.filter_first_per_process = true; process_record(&mut c, include_bytes!("testdata/record-nscd.txt")).unwrap(); assert!( - events.borrow().is_empty(), + !events + .borrow() + .iter() + .any(|e| &e.id == "1705071450.879:29498378"), "nscd connect event should be filtered" ) } @@ -1550,10 +1586,15 @@ mod test { }; let s2 = Settings { filter_keys: [b"fork".to_vec()].into(), + filter_first_per_process: true, ..s1.clone() }; + let s3 = Settings { + filter_first_per_process: false, // default in 0.6.2+ + ..s2.clone() + }; - for (n, s) in [s1, s2].iter().enumerate() { + for (n, s) in [s1, s2, s3].iter().enumerate() { let events: Rc>> = Rc::new(RefCell::new(vec![])); let mut c = Coalesce::new(mk_emit_vec(&events)); @@ -1564,27 +1605,56 @@ mod test { let events = events.borrow(); - let mut ids = vec![ + let mut present_and_label = vec![ "1682609045.526:29238", "1682609045.530:29242", "1682609045.530:29244", "1682609045.534:29245", ]; - if n == 0 { - ids.extend([ - "1682609045.530:29240", - "1682609045.530:29241", - "1682609045.530:29243", - ]); - } + let mut absent = vec![]; + match n { + 0 => { + present_and_label.extend([ + "1682609045.530:29239", + "1682609045.530:29240", + "1682609045.530:29241", + "1682609045.530:29243", + ]); + } + 1 => { + absent.extend([ + "1682609045.526:29237", + "1682609045.530:29239", + "1682609045.530:29240", + "1682609045.530:29241", + "1682609045.530:29243", + ]); + } + 2 => { + // fork = first event in pid=71506 + present_and_label.extend(["1682609045.530:29241"]); + + absent.extend([ + "1682609045.530:29239", + "1682609045.530:29240", + "1682609045.530:29243", + ]); + } + _ => {} + }; - for id in ids { + for id in present_and_label { let event = find_event(&events, id).expect(&format!("Did not find {id}")); assert!( event_to_json(&event).contains(r#""LABELS":["test-script"]"#), "{id} was not labelled correctly." ); } + for id in absent { + if find_event(&events, id).is_some() { + panic!("Found {id} though it should have been filtered."); + } + } } } diff --git a/src/config.rs b/src/config.rs index 6affb9f..1e95ebf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -165,6 +165,8 @@ pub struct Filter { pub filter_null_keys: bool, #[serde(default, rename = "filter-action")] pub filter_action: FilterAction, + #[serde(default = "true_value", rename = "keep-first-per-process")] + pub keep_first_per_process: bool, } #[derive(Debug, Serialize, Default)] @@ -346,6 +348,7 @@ impl Config { .collect(), filter_null_keys: self.filter.filter_null_keys, filter_raw_lines: self.filter.filter_raw_lines.clone(), + filter_first_per_process: !self.filter.keep_first_per_process, } } } diff --git a/src/testdata/record-syscall-key.txt b/src/testdata/record-syscall-key.txt index b1a2b11..c0ad91c 100644 --- a/src/testdata/record-syscall-key.txt +++ b/src/testdata/record-syscall-key.txt @@ -1,3 +1,6 @@ -type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this" -type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="this-too" +type=SYSCALL msg=audit(1628602815.266:2365): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this" +type=EOE msg=audit(1628602815.266:2365): +type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=0 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this" type=EOE msg=audit(1628602815.266:2366): +type=SYSCALL msg=audit(1628602815.266:2367): arch=c000003e syscall=0 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="this-too" +type=EOE msg=audit(1628602815.266:2367): diff --git a/src/testdata/record-syscall-nullkey.txt b/src/testdata/record-syscall-nullkey.txt index 071366c..1236cee 100644 --- a/src/testdata/record-syscall-nullkey.txt +++ b/src/testdata/record-syscall-nullkey.txt @@ -1,10 +1,10 @@ -type=PROCTITLE msg=audit(1678282381.452:102337): proctitle="(systemd)" +type=SYSCALL msg=audit(1678282381.452:102336): arch=c000003e syscall=59 success=yes exit=5 a0=9 a1=7ffd4ac563d1 a2=5 a3=0 items=0 ppid=1 pid=3489504 auid=34005 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15589 comm="(systemd)" exe="/usr/lib/systemd/systemd" subj=system_u:system_r:init_t:s0 key=(null) +type=EOE msg=audit(1678282381.452:102336): + type=SYSCALL msg=audit(1678282381.452:102337): arch=c000003e syscall=1 success=yes exit=5 a0=9 a1=7ffd4ac563d1 a2=5 a3=0 items=0 ppid=1 pid=3489504 auid=34005 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15589 comm="(systemd)" exe="/usr/lib/systemd/systemd" subj=system_u:system_r:init_t:s0 key=(null) type=EOE msg=audit(1678282381.452:102337): -type=PROCTITLE msg=audit(1678282320.958:102262): proctitle=536f6d6552616e646f6d50726f63657373 -type=SYSCALL msg=audit(1678282320.958:102262): arch=c000003e syscall=1 success=yes exit=5 a0=3 a1=7ffd9f4453e0 a2=5 a3=0 items=0 ppid=8750 pid=3483623 auid=34025 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15584 comm="sshd" exe="/bin/sshd" subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 key=(null) -type=EOE msg=audit(1678282320.958:102262): + type=PROCTITLE msg=audit(1678283440.683:225): proctitle=536f6d6552616e646f6d50726f63657373 type=PATH msg=audit(1678283440.683:225): item=0 name="/proc/2414/root/usr/bin/su" inode=156161 dev=fd:00 mode=0104755 ouid=0 ogid=0 rdev=00:00 obj=system_u:object_r:su_exec_t:s0 objtype=NORMAL cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0 type=SYSCALL msg=audit(1678283440.683:225): arch=c000003e syscall=4 success=yes exit=0 a0=7edd0caa2e7e0 a1=7345b64adba0 a2=7ff9874adba0 a3=feefeffefefefeff items=1 ppid=816 pid=818 auid=4292467295 uid=502 gid=502 euid=502 suid=502 fsuid=502 egid=502 sgid=502 fsgid=502 tty=(none) ses=4296967295 comm="cat" exe="/usr/bin/cat" subj=system_u:system_r:system_t:s0 key=(null) -type=EOE msg=audit(1678283440.683:225): \ No newline at end of file +type=EOE msg=audit(1678283440.683:225):