From b051c2d79eabd309b7e537142beda892f8a1efea Mon Sep 17 00:00:00 2001 From: Dario Nieuwenhuis Date: Thu, 2 May 2024 16:16:52 +0200 Subject: [PATCH] ci: test with more features, run fuzzers. --- ci.sh | 33 +++++++++++++++++++++++---- fuzz/fuzz_targets/file.rs | 6 +++-- src/file.rs | 47 ++++++++++++++++++++++----------------- src/lib.rs | 1 + 4 files changed, 61 insertions(+), 26 deletions(-) diff --git a/ci.sh b/ci.sh index 3ede23e..44c7ab7 100755 --- a/ci.sh +++ b/ci.sh @@ -2,12 +2,37 @@ set -euxo pipefail +cargo install cargo-fuzz + export RUSTFLAGS=-Dwarnings cargo fmt --check -cargo test --release -cargo test --release --features crc -cargo clippy -cargo clippy --features crc + +FEATURESET=( + ekv/page-size-256,ekv/max-page-count-64,ekv/max-value-size-128,ekv/scratch-page-count-4,ekv/align-1 + ekv/page-size-256,ekv/max-page-count-256,ekv/max-value-size-1024,ekv/scratch-page-count-8,ekv/align-2 + ekv/page-size-1024,ekv/max-page-count-16,ekv/max-value-size-16,ekv/scratch-page-count-0 + ekv/page-size-128,ekv/max-page-count-32,ekv/max-value-size-16,ekv/scratch-page-count-0 + ekv/page-size-128,ekv/max-page-count-32,ekv/max-value-size-16,ekv/scratch-page-count-0,ekv/crc +) + +for FEATURES in ${FEATURESET[@]}; do + cargo test --release --features $FEATURES + cargo clippy --features $FEATURES + + # Run `file` fuzzer + cargo fuzz run --sanitizer none -j$(nproc) file --features $FEATURES -- -max_total_time=30 + + # Run `ops` fuzzer + cargo fuzz run --sanitizer none -j$(nproc) ops --features $FEATURES -- -max_total_time=30 + + # Run `read` fuzzer. Seed it with a flash image with stuff in it. + cargo run --release --example smoke --features $FEATURES + rm -rf fuzz/corpus/read + mkdir -p fuzz/corpus/read + mv out.bin fuzz/corpus/read + cargo fuzz run --sanitizer none -j$(nproc) read --features $FEATURES -- -max_total_time=30 +done + (cd examples/nrf; cargo fmt --check; cargo build --release --features defmt) (cd examples/rp2040; cargo fmt --check; cargo build --release) diff --git a/fuzz/fuzz_targets/file.rs b/fuzz/fuzz_targets/file.rs index 86e2ed7..9193db1 100644 --- a/fuzz/fuzz_targets/file.rs +++ b/fuzz/fuzz_targets/file.rs @@ -8,6 +8,7 @@ use ekv::flash::MemFlash; use ekv::page::PageReader; use libfuzzer_sys::arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; +use log::info; const MAX_LEN: usize = 1024; @@ -60,13 +61,14 @@ async fn fuzz_inner(ops: Input, dump: bool) { let mut trunc_offs: usize = 0; for op in ops.ops { + info!("========== OP: {:?}", op); match op { Op::Append { len } => { if len > MAX_LEN { continue; } - if (m.used_pages() + 3) * PAGE_SIZE + len >= MAX_PAGE_COUNT * PAGE_SIZE { + if (m.used_pages() + 4) * PAGE_SIZE + len >= MAX_PAGE_COUNT * PAGE_SIZE { continue; } @@ -127,7 +129,7 @@ async fn fuzz_inner(ops: Input, dump: bool) { } // Linear search - loop { + while !found { let mut got_id = [0u8; 4]; match s.reader().read(&mut m, &mut got_id).await { Ok(()) => {} diff --git a/src/file.rs b/src/file.rs index 224ae1f..9934999 100644 --- a/src/file.rs +++ b/src/file.rs @@ -1587,6 +1587,9 @@ mod tests { use crate::flash::MemFlash; use crate::types::RawPageID; + /// Big amount of data that it's reasonable to write to a file. + const BIG_DATA_SIZE: usize = PAGE_SIZE * MAX_PAGE_COUNT / 2; + fn page(p: RawPageID) -> PageID { PageID::from_raw(p).unwrap() } @@ -1627,7 +1630,7 @@ mod tests { m.format().await.unwrap(); m.mount(&mut pr).await.unwrap(); - let data = dummy_data(23456); + let data = dummy_data(BIG_DATA_SIZE); let mut w = m.write(&mut pr, 0).await.unwrap(); w.write(&mut m, &data).await.unwrap(); @@ -2652,7 +2655,7 @@ mod tests { m.mount(&mut pr).await.unwrap(); let mut w = m.write(&mut pr, 1).await.unwrap(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; BIG_DATA_SIZE]).await.unwrap(); w.record_end(); m.commit(&mut w).await.unwrap(); @@ -2671,10 +2674,12 @@ mod tests { m.format().await.unwrap(); m.mount(&mut pr).await.unwrap(); + const N: usize = BIG_DATA_SIZE / 2; + let mut w = m.write(&mut pr, 1).await.unwrap(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; N]).await.unwrap(); w.record_end(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; N]).await.unwrap(); w.record_end(); m.commit(&mut w).await.unwrap(); @@ -2682,7 +2687,7 @@ mod tests { // Seek left let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), false); assert_eq!(s.reader().offset(&mut m), 0); @@ -2690,10 +2695,10 @@ mod tests { // Seek right let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Right).await.unwrap(), false); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); } #[test_log::test(tokio::test)] @@ -2704,12 +2709,14 @@ mod tests { m.format().await.unwrap(); m.mount(&mut pr).await.unwrap(); + const N: usize = BIG_DATA_SIZE / 3; + let mut w = m.write(&mut pr, 1).await.unwrap(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; N]).await.unwrap(); w.record_end(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; N]).await.unwrap(); w.record_end(); - w.write(&mut m, &[0x00; 4348]).await.unwrap(); + w.write(&mut m, &[0x00; N]).await.unwrap(); w.record_end(); m.commit(&mut w).await.unwrap(); @@ -2717,10 +2724,10 @@ mod tests { // Seek left let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 8696); + assert_eq!(s.reader().offset(&mut m), N * 2); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), false); assert_eq!(s.reader().offset(&mut m), 0); @@ -2728,10 +2735,10 @@ mod tests { // Seek less left let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 8696); + assert_eq!(s.reader().offset(&mut m), N * 2); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), false); assert_eq!(s.reader().offset(&mut m), 0); @@ -2739,21 +2746,21 @@ mod tests { // Seek middle let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 8696); + assert_eq!(s.reader().offset(&mut m), N * 2); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Left).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Right).await.unwrap(), false); - assert_eq!(s.reader().offset(&mut m), 4348); + assert_eq!(s.reader().offset(&mut m), N); // Seek right let mut s = FileSearcher::new(m.read(&mut pr, 1)); assert_eq!(s.start(&mut m).await.unwrap(), true); - assert_eq!(s.reader().offset(&mut m), 8696); + assert_eq!(s.reader().offset(&mut m), N * 2); s.reader().read(&mut m, &mut buf).await.unwrap(); assert_eq!(s.seek(&mut m, SeekDirection::Right).await.unwrap(), false); - assert_eq!(s.reader().offset(&mut m), 8696); + assert_eq!(s.reader().offset(&mut m), N * 2); } #[test_log::test(tokio::test)] @@ -2814,7 +2821,7 @@ mod tests { m.format().await.unwrap(); m.mount(&mut pr).await.unwrap(); - let count: u32 = 20000 / 4; + let count: u32 = (BIG_DATA_SIZE / 4).try_into().unwrap(); let mut w = m.write(&mut pr, 1).await.unwrap(); for i in 1..=count { diff --git a/src/lib.rs b/src/lib.rs index 60add50..8145dd8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ #![allow(async_fn_in_trait)] #![allow(clippy::large_enum_variant)] #![allow(clippy::new_without_default)] +#![allow(clippy::modulo_one)] // needed when ALIGN=1 // the `_test` feature makes public more stuff, causing bogus warnings. #![cfg_attr(not(feature = "_test"), warn(missing_docs))]