Make filtering first event per process optional

The first observed event in a process is special because it is used to
uniquely identify that process when enriching PIDs. It is therefore
useful to keep this first event in the main log.

etc/laurel/config.toml

@@ -158,6 +158,12 @@ filter-null-keys = false
 #     "^type=PATH msg=\\S*? item=\\S*? name=\"/var/run/nscd[.]sock\" "
 # ]
 
+# Keep the first event observed for any given process even if it would
+# be filtered otherwise. This should only be turned off if
+# reproducible process tracking or process tree reconstruction is not
+# required.
+# keep-first-per-processes = true
+
 # What to do with filtered events? "drop" or "log" to the filterlog
 # defined above.
 filter-action = "drop"

man/laurel.8.md

@@ -175,6 +175,10 @@ using them for internal processing such as process tracking.
   that contain such lines are then filtered. Default: empty
 - `filter-action`: What to do with filtered events? `drop` or `log` to the
   filterlog defined above.
+- `keep-first-per-process`: Keep the first event observed for any
+  given process even if it would be filtered otherwise. This should
+  only be turned off if reproducible process tracking or process tree
+  reconstruction is not required. Default: true
 
 # SIGNALS
 

src/coalesce.rs

@@ -49,6 +49,7 @@ pub struct Settings {
     pub filter_labels: HashSet<Vec<u8>>,
     pub filter_null_keys: bool,
     pub filter_raw_lines: regex::bytes::RegexSet,
+    pub filter_first_per_process: bool,
 }
 
 impl Default for Settings {
@@ -75,6 +76,7 @@ impl Default for Settings {
             filter_labels: HashSet::new(),
             filter_null_keys: false,
             filter_raw_lines: regex::bytes::RegexSet::empty(),
+            filter_first_per_process: false,
         }
     }
 }
@@ -656,6 +658,7 @@ impl<'a, 'ev> Coalesce<'a, 'ev> {
         let mut syscall_name: Option<&'static str> = None;
 
         let mut syscall_is_exec = false;
+        let mut force_keep = false;
         let mut current_process: Option<Process> = None;
         let mut parent: Option<Process> = None;
 
@@ -747,11 +750,12 @@ impl<'a, 'ev> Coalesce<'a, 'ev> {
                         // Process entry.
                         syscall_is_exec = sn.contains("execve");
                         parent = self.processes.get_or_retrieve(proc.ppid).cloned();
-                        let pr = if !syscall_is_exec {
-                            self.processes.get_or_retrieve(proc.pid)
+                        let pr;
+                        if syscall_is_exec {
+                            pr = None;
                         } else {
-                            None
-                        };
+                            pr = self.processes.get_or_retrieve(proc.pid);
+                        }
                         match pr {
                             Some(pr) if proc.ppid == pr.ppid && proc.exe == pr.exe => {
                                 // existing, plausible process in table
@@ -768,6 +772,10 @@ impl<'a, 'ev> Coalesce<'a, 'ev> {
                             }
                             _ => {
                                 // first syscall in new process
+                                if !self.settings.filter_first_per_process {
+                                    ev.filter = false;
+                                    force_keep = true;
+                                }
                                 proc.key = ProcessKey::Event(ev.id);
                                 if let Some(pa) = &parent {
                                     proc.parent = Some(pa.key);
@@ -805,13 +813,13 @@ impl<'a, 'ev> Coalesce<'a, 'ev> {
             }
 
             if let Some(key) = &key {
-                if self.settings.filter_keys.contains(key.as_ref()) {
+                if !force_keep && self.settings.filter_keys.contains(key.as_ref()) {
                     ev.filter = true;
                 }
                 if self.settings.proc_label_keys.contains(key.as_ref()) {
                     proc.labels.insert(key.to_vec());
                 }
-            } else if self.settings.filter_null_keys {
+            } else if !force_keep && self.settings.filter_null_keys {
                 ev.filter = true;
             }
 
@@ -866,10 +874,11 @@ impl<'a, 'ev> Coalesce<'a, 'ev> {
         // filter early on labels
         if let Some(proc) = &current_process {
             self.processes.set_labels(&proc.key, &proc.labels);
-            if proc
-                .labels
-                .iter()
-                .any(|x| self.settings.filter_labels.contains(x))
+            if !force_keep
+                && proc
+                    .labels
+                    .iter()
+                    .any(|x| self.settings.filter_labels.contains(x))
             {
                 ev.filter = true;
             }
@@ -1398,7 +1407,19 @@ mod test {
         c.settings.filter_keys.insert(Vec::from(&b"this-too"[..]));
         process_record(&mut c, include_bytes!("testdata/record-syscall-key.txt"))?;
         drop(c);
-        assert!(events.borrow().is_empty());
+        // fist event for process -> don't filter
+        assert!(events
+            .borrow()
+            .iter()
+            .any(|e| &e.id == "1628602815.266:2365"));
+        assert!(!events
+            .borrow()
+            .iter()
+            .any(|e| &e.id == "1628602815.266:2366"));
+        assert!(!events
+            .borrow()
+            .iter()
+            .any(|e| &e.id == "1628602815.266:2367"));
 
         let mut c = Coalesce::new(mk_emit_vec(&events));
         c.settings.filter_null_keys = true;
@@ -1407,7 +1428,17 @@ mod test {
             include_bytes!("testdata/record-syscall-nullkey.txt"),
         )?;
         drop(c);
-        assert!(events.borrow().is_empty());
+
+        // not first event for process -> filter
+        assert!(!events
+            .borrow()
+            .iter()
+            .any(|e| &e.id == "1678282381.452:102337"));
+        // fist event for process -> don't filter
+        assert!(events
+            .borrow()
+            .iter()
+            .any(|e| &e.id == "1678283440.683:225"));
 
         let mut c = Coalesce::new(mk_emit_vec(&events));
         c.settings
@@ -1425,6 +1456,7 @@ mod test {
         let ec: Rc<RefCell<Option<Event>>> = Rc::new(RefCell::new(None));
 
         let mut c = Coalesce::new(mk_emit(&ec));
+        c.settings.filter_first_per_process = true;
         c.settings
             .proc_label_keys
             .insert(Vec::from(&b"software_mgmt"[..]));
@@ -1464,11 +1496,15 @@ mod test {
             "^type=SOCKADDR (?:node=\\$*? )?msg=audit\\(\\S*?\\): saddr=01002F7661722F72756E2F6E7363642F736F636B657400",
         ])
         .expect("failed to compile regex");
+        c.settings.filter_first_per_process = true;
 
         process_record(&mut c, include_bytes!("testdata/record-nscd.txt")).unwrap();
 
         assert!(
-            events.borrow().is_empty(),
+            !events
+                .borrow()
+                .iter()
+                .any(|e| &e.id == "1705071450.879:29498378"),
             "nscd connect event should be filtered"
         )
     }
@@ -1550,10 +1586,15 @@ mod test {
         };
         let s2 = Settings {
             filter_keys: [b"fork".to_vec()].into(),
+            filter_first_per_process: true,
             ..s1.clone()
         };
+        let s3 = Settings {
+            filter_first_per_process: false, // default in 0.6.2+
+            ..s2.clone()
+        };
 
-        for (n, s) in [s1, s2].iter().enumerate() {
+        for (n, s) in [s1, s2, s3].iter().enumerate() {
             let events: Rc<RefCell<Vec<Event>>> = Rc::new(RefCell::new(vec![]));
             let mut c = Coalesce::new(mk_emit_vec(&events));
 
@@ -1564,27 +1605,56 @@ mod test {
 
             let events = events.borrow();
 
-            let mut ids = vec![
+            let mut present_and_label = vec![
                 "1682609045.526:29238",
                 "1682609045.530:29242",
                 "1682609045.530:29244",
                 "1682609045.534:29245",
             ];
-            if n == 0 {
-                ids.extend([
-                    "1682609045.530:29240",
-                    "1682609045.530:29241",
-                    "1682609045.530:29243",
-                ]);
-            }
+            let mut absent = vec![];
+            match n {
+                0 => {
+                    present_and_label.extend([
+                        "1682609045.530:29239",
+                        "1682609045.530:29240",
+                        "1682609045.530:29241",
+                        "1682609045.530:29243",
+                    ]);
+                }
+                1 => {
+                    absent.extend([
+                        "1682609045.526:29237",
+                        "1682609045.530:29239",
+                        "1682609045.530:29240",
+                        "1682609045.530:29241",
+                        "1682609045.530:29243",
+                    ]);
+                }
+                2 => {
+                    // fork = first event in pid=71506
+                    present_and_label.extend(["1682609045.530:29241"]);
+
+                    absent.extend([
+                        "1682609045.530:29239",
+                        "1682609045.530:29240",
+                        "1682609045.530:29243",
+                    ]);
+                }
+                _ => {}
+            };
 
-            for id in ids {
+            for id in present_and_label {
                 let event = find_event(&events, id).expect(&format!("Did not find {id}"));
                 assert!(
                     event_to_json(&event).contains(r#""LABELS":["test-script"]"#),
                     "{id} was not labelled correctly."
                 );
             }
+            for id in absent {
+                if find_event(&events, id).is_some() {
+                    panic!("Found {id} though it should have been filtered.");
+                }
+            }
         }
     }
 

src/config.rs

@@ -165,6 +165,8 @@ pub struct Filter {
     pub filter_null_keys: bool,
     #[serde(default, rename = "filter-action")]
     pub filter_action: FilterAction,
+    #[serde(default = "true_value", rename = "keep-first-per-process")]
+    pub keep_first_per_process: bool,
 }
 
 #[derive(Debug, Serialize, Default)]
@@ -346,6 +348,7 @@ impl Config {
                 .collect(),
             filter_null_keys: self.filter.filter_null_keys,
             filter_raw_lines: self.filter.filter_raw_lines.clone(),
+            filter_first_per_process: !self.filter.keep_first_per_process,
         }
     }
 }

src/testdata/record-syscall-key.txt

@@ -1,3 +1,6 @@
-type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this"
-type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="this-too"
+type=SYSCALL msg=audit(1628602815.266:2365): arch=c000003e syscall=59 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this"
+type=EOE msg=audit(1628602815.266:2365):
+type=SYSCALL msg=audit(1628602815.266:2366): arch=c000003e syscall=0 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="filter-this"
 type=EOE msg=audit(1628602815.266:2366):
+type=SYSCALL msg=audit(1628602815.266:2367): arch=c000003e syscall=0 success=yes exit=0 a0=2557470 a1=247b510 a2=2565820 a3=5bb items=2 ppid=3193 pid=6382 auid=1000 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=pts0 ses=1 comm="cat" exe="/usr/bin/cat" key="this-too"
+type=EOE msg=audit(1628602815.266:2367):

src/testdata/record-syscall-nullkey.txt

@@ -1,10 +1,10 @@
-type=PROCTITLE msg=audit(1678282381.452:102337): proctitle="(systemd)"
+type=SYSCALL msg=audit(1678282381.452:102336): arch=c000003e syscall=59 success=yes exit=5 a0=9 a1=7ffd4ac563d1 a2=5 a3=0 items=0 ppid=1 pid=3489504 auid=34005 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15589 comm="(systemd)" exe="/usr/lib/systemd/systemd" subj=system_u:system_r:init_t:s0 key=(null)
+type=EOE msg=audit(1678282381.452:102336): 
+
 type=SYSCALL msg=audit(1678282381.452:102337): arch=c000003e syscall=1 success=yes exit=5 a0=9 a1=7ffd4ac563d1 a2=5 a3=0 items=0 ppid=1 pid=3489504 auid=34005 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15589 comm="(systemd)" exe="/usr/lib/systemd/systemd" subj=system_u:system_r:init_t:s0 key=(null)
 type=EOE msg=audit(1678282381.452:102337): 
-type=PROCTITLE msg=audit(1678282320.958:102262): proctitle=536f6d6552616e646f6d50726f63657373
-type=SYSCALL msg=audit(1678282320.958:102262): arch=c000003e syscall=1 success=yes exit=5 a0=3 a1=7ffd9f4453e0 a2=5 a3=0 items=0 ppid=8750 pid=3483623 auid=34025 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=15584 comm="sshd" exe="/bin/sshd" subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 key=(null)
-type=EOE msg=audit(1678282320.958:102262): 
+
 type=PROCTITLE msg=audit(1678283440.683:225): proctitle=536f6d6552616e646f6d50726f63657373
 type=PATH msg=audit(1678283440.683:225): item=0 name="/proc/2414/root/usr/bin/su" inode=156161 dev=fd:00 mode=0104755 ouid=0 ogid=0 rdev=00:00 obj=system_u:object_r:su_exec_t:s0 objtype=NORMAL cap_fp=0000000000000000 cap_fi=0000000000000000 cap_fe=0 cap_fver=0
 type=SYSCALL msg=audit(1678283440.683:225): arch=c000003e syscall=4 success=yes exit=0 a0=7edd0caa2e7e0 a1=7345b64adba0 a2=7ff9874adba0 a3=feefeffefefefeff items=1 ppid=816 pid=818 auid=4292467295 uid=502 gid=502 euid=502 suid=502 fsuid=502 egid=502 sgid=502 fsgid=502 tty=(none) ses=4296967295 comm="cat" exe="/usr/bin/cat" subj=system_u:system_r:system_t:s0 key=(null)
-type=EOE msg=audit(1678283440.683:225):
+type=EOE msg=audit(1678283440.683:225):