refactor: Loro import function should return LoroEncodeError (#487)

* refactor: Loro import function should return LoroEncodeError - Change return type - Add support for handling potential errors: - Invalid or non-existent frontiers - Exporting old snapshot format from trimmed snapshot doc - Improve compatibility with trimmed docs * fix: add tests and fix a few places that need to return err
loro-dev · Oct 2, 2024 · 09a004e · 09a004e
1 parent 31bc451
commit 09a004e
Show file tree

Hide file tree

Showing 42 changed files with 364 additions and 175 deletions.
diff --git a/crates/examples/benches/bench_text.rs b/crates/examples/benches/bench_text.rs
@@ -43,7 +43,7 @@ fn bench_text(c: &mut Criterion) {
                 doc
             },
             |doc| {
-                doc.export(loro::ExportMode::Snapshot);
+                doc.export(loro::ExportMode::Snapshot).unwrap();
             },
             criterion::BatchSize::SmallInput,
         )
@@ -53,7 +53,7 @@ fn bench_text(c: &mut Criterion) {
         b.iter_batched(
             || apply_text_actions(&actions, 1),
             |doc| {
-                doc.export(loro::ExportMode::Snapshot);
+                doc.export(loro::ExportMode::Snapshot).unwrap();
             },
             criterion::BatchSize::SmallInput,
         )
@@ -68,7 +68,7 @@ fn bench_text(c: &mut Criterion) {
                 }
                 if doc_snapshot.get().is_none() {
                     let doc = doc.get().unwrap();
-                    let snapshot = doc.export(loro::ExportMode::Snapshot);
+                    let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
                     println!("B4 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_snapshot.set(snapshot).unwrap();
                 }
@@ -87,7 +87,7 @@ fn bench_text(c: &mut Criterion) {
             || {
                 if doc_x100_snapshot.get().is_none() {
                     let doc = apply_text_actions(&actions, 100);
-                    let snapshot = doc.export(loro::ExportMode::Snapshot);
+                    let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
                     println!("B4x100 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_x100_snapshot.set(snapshot).unwrap();
                 }
@@ -107,7 +107,7 @@ fn bench_text(c: &mut Criterion) {
             || {
                 if doc_x100_snapshot.get().is_none() {
                     let doc = apply_text_actions(&actions, 100);
-                    let snapshot = doc.export(loro::ExportMode::Snapshot);
+                    let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
                     println!("B4x100 fast_snapshot size: {:?}", ByteSize(snapshot.len()));
                     doc_x100_snapshot.set(snapshot).unwrap();
                 }

diff --git a/crates/examples/examples/init_sheet.rs b/crates/examples/examples/init_sheet.rs
@@ -8,7 +8,7 @@ pub fn main() {
     println!("init_duration {}", init_duration);
 
     let start = Instant::now();
-    let snapshot = doc.export(loro::ExportMode::Snapshot);
+    let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
     let duration = start.elapsed().as_secs_f64() * 1000.;
     println!("export duration {} size={}", duration, snapshot.len());
 }
diff --git a/crates/examples/examples/large_movable_list.rs b/crates/examples/examples/large_movable_list.rs
@@ -65,7 +65,7 @@ pub fn main() {
     println!("Time cost {:?}", start.elapsed());
 
     let start = Instant::now();
-    let updates = doc.export(loro::ExportMode::all_updates());
+    let updates = doc.export(loro::ExportMode::all_updates()).unwrap();
     println!("Export updates time cost {:?}", start.elapsed());
     let start = Instant::now();
     let doc2 = LoroDoc::new();

diff --git a/crates/examples/examples/list.rs b/crates/examples/examples/list.rs
@@ -61,10 +61,18 @@ pub fn main() {
                 random_insert(&mut list_a, 100, i);
                 random_insert(&mut list_b, 100, i);
                 doc_a
-                    .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                    .import(
+                        &doc_b
+                            .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
                 doc_b
-                    .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                    .import(
+                        &doc_a
+                            .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
             }
 
@@ -111,10 +119,18 @@ pub fn main() {
                 random_insert(&mut list_a, 100, i);
                 random_insert(&mut list_b, 100, i);
                 doc_a
-                    .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                    .import(
+                        &doc_b
+                            .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
                 doc_b
-                    .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                    .import(
+                        &doc_a
+                            .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
             }
 
@@ -128,19 +144,35 @@ pub fn main() {
             random_insert(&mut list_a, 1000, 0);
             random_insert(&mut list_b, 1000, 0);
             doc_a
-                .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                .import(
+                    &doc_b
+                        .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                        .unwrap(),
+                )
                 .unwrap();
             doc_b
-                .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                .import(
+                    &doc_a
+                        .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                        .unwrap(),
+                )
                 .unwrap();
             for i in 0..1000 {
                 random_set(&mut list_a, 100, i);
                 random_set(&mut list_b, 100, i);
                 doc_a
-                    .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                    .import(
+                        &doc_b
+                            .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
                 doc_b
-                    .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                    .import(
+                        &doc_a
+                            .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
             }
 
@@ -154,19 +186,35 @@ pub fn main() {
             random_insert(&mut list_a, 1000, 0);
             random_insert(&mut list_b, 1000, 0);
             doc_a
-                .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                .import(
+                    &doc_b
+                        .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                        .unwrap(),
+                )
                 .unwrap();
             doc_b
-                .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                .import(
+                    &doc_a
+                        .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                        .unwrap(),
+                )
                 .unwrap();
             for i in 0..1000 {
                 random_move(&mut list_a, 100, i);
                 random_move(&mut list_b, 100, i);
                 doc_a
-                    .import(&doc_b.export(loro::ExportMode::updates(&doc_a.oplog_vv())))
+                    .import(
+                        &doc_b
+                            .export(loro::ExportMode::updates(&doc_a.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
                 doc_b
-                    .import(&doc_a.export(loro::ExportMode::updates(&doc_b.oplog_vv())))
+                    .import(
+                        &doc_a
+                            .export(loro::ExportMode::updates(&doc_b.oplog_vv()))
+                            .unwrap(),
+                    )
                     .unwrap();
             }
 
@@ -186,11 +234,11 @@ fn run(name: &'static str, apply_task: impl FnOnce() -> LoroDoc) -> BenchResult
     let apply_duration = start.elapsed().as_secs_f64() * 1000.;
 
     let start = Instant::now();
-    let snapshot = doc.export(loro::ExportMode::Snapshot);
+    let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
     let encode_snapshot_duration = start.elapsed().as_secs_f64() * 1000.;
 
     let start = Instant::now();
-    let updates = doc.export(loro::ExportMode::all_updates());
+    let updates = doc.export(loro::ExportMode::all_updates()).unwrap();
     let encode_update_duration = start.elapsed().as_secs_f64() * 1000.;
 
     let start = Instant::now();

diff --git a/crates/examples/examples/outliner.rs b/crates/examples/examples/outliner.rs
@@ -42,10 +42,10 @@ pub fn main() {
 
     println!(
         "Updates size: {}",
-        ByteSize(doc.export(loro::ExportMode::Snapshot).len())
+        ByteSize(doc.export(loro::ExportMode::Snapshot).unwrap().len())
     );
     let snapshot = doc.export(loro::ExportMode::Snapshot);
-    println!("Snapshot size: {}", ByteSize(snapshot.len()));
+    println!("Snapshot size: {}", ByteSize(snapshot.unwrap().len()));
     doc.with_oplog(|oplog| {
         println!(
             "Change store kv size: {}",
@@ -72,8 +72,12 @@ pub fn main() {
             }
         }
 
-        doc.import(&new_doc.export(loro::ExportMode::updates(&doc.oplog_vv())))
-            .unwrap();
+        doc.import(
+            &new_doc
+                .export(loro::ExportMode::updates(&doc.oplog_vv()))
+                .unwrap(),
+        )
+        .unwrap();
     }
 
     println!("Time taken to move {} nodes: {:?}", n * k, start.elapsed());
@@ -100,10 +104,10 @@ pub fn main() {
 
     println!(
         "Updates size: {}",
-        ByteSize(doc.export(loro::ExportMode::all_updates()).len())
+        ByteSize(doc.export(loro::ExportMode::all_updates()).unwrap().len())
     );
     let snapshot = doc.export(loro::ExportMode::Snapshot);
-    println!("Snapshot size: {}", ByteSize(snapshot.len()));
+    println!("Snapshot size: {}", ByteSize(snapshot.unwrap().len()));
     doc.compact_change_store();
     doc.with_oplog(|oplog| {
         println!(

diff --git a/crates/examples/examples/time_tracker.rs b/crates/examples/examples/time_tracker.rs
@@ -50,12 +50,12 @@ pub fn main() {
     println!("total_time: {}", total_time);
     println!("mem: {}", get_mem_usage());
     let snapshot = doc.export(loro::ExportMode::Snapshot);
-    println!("Snapshot Size {}", ByteSize(snapshot.len()));
+    println!("Snapshot Size {}", ByteSize(snapshot.unwrap().len()));
     println!("mem: {}", get_mem_usage());
     let trimmed_snapshot = doc.export(loro::ExportMode::trimmed_snapshot(&doc.oplog_frontiers()));
     println!(
         "GC Shallow Snapshot Size {}",
-        ByteSize(trimmed_snapshot.len())
+        ByteSize(trimmed_snapshot.unwrap().len())
     );
     println!("mem: {}", get_mem_usage());
 

diff --git a/crates/examples/src/utils.rs b/crates/examples/src/utils.rs
@@ -38,7 +38,7 @@ pub fn bench_fast_snapshot(doc: &LoroDoc) {
     {
         println!("======== New snapshot mode =========");
         let start = Instant::now();
-        let snapshot = doc.export(loro::ExportMode::Snapshot);
+        let snapshot = doc.export(loro::ExportMode::Snapshot).unwrap();
         let elapsed = start.elapsed();
         println!("Fast Snapshot size: {}", ByteSize(snapshot.len()));
         println!("Export fast snapshot time: {:?}", elapsed);
@@ -85,7 +85,9 @@ pub fn bench_fast_snapshot(doc: &LoroDoc) {
     {
         println!("======== New snapshot mode with GC =========");
         let start = Instant::now();
-        let snapshot = doc.export(loro::ExportMode::trimmed_snapshot(&doc.oplog_frontiers()));
+        let snapshot = doc
+            .export(loro::ExportMode::trimmed_snapshot(&doc.oplog_frontiers()))
+            .unwrap();
         let elapsed = start.elapsed();
         println!("Fast Snapshot size: {}", ByteSize(snapshot.len()));
         println!("Export fast snapshot time: {:?}", elapsed);

diff --git a/crates/fuzz/src/actor.rs b/crates/fuzz/src/actor.rs
@@ -232,7 +232,7 @@ impl Actor {
                 let new_doc = LoroDoc::new();
                 info_span!("FuzzCheckoutCreatingNewSnapshotDoc",).in_scope(|| {
                     new_doc
-                        .import(&self.loro.export(loro::ExportMode::Snapshot))
+                        .import(&self.loro.export(loro::ExportMode::Snapshot).unwrap())
                         .unwrap();
                     assert_eq!(new_doc.get_deep_value(), self.loro.get_deep_value());
                 });

diff --git a/crates/fuzz/src/crdt_fuzzer.rs b/crates/fuzz/src/crdt_fuzzer.rs
@@ -224,12 +224,12 @@ impl CRDTFuzzer {
                     2 => {
                         info_span!("FastSnapshot", from = i, to = j).in_scope(|| {
                             b_doc
-                                .import(&a_doc.export(loro::ExportMode::Snapshot))
+                                .import(&a_doc.export(loro::ExportMode::Snapshot).unwrap())
                                 .unwrap();
                         });
                         info_span!("FastSnapshot", from = j, to = i).in_scope(|| {
                             a_doc
-                                .import(&b_doc.export(loro::ExportMode::Snapshot))
+                                .import(&b_doc.export(loro::ExportMode::Snapshot).unwrap())
                                 .unwrap();
                         });
                     }
@@ -383,7 +383,7 @@ pub fn test_multi_sites_with_gc(
                     let bytes = fuzzer.actors[1]
                         .loro
                         .export(loro::ExportMode::trimmed_snapshot(&f));
-                    fuzzer.actors[0].loro.import(&bytes).unwrap();
+                    fuzzer.actors[0].loro.import(&bytes.unwrap()).unwrap();
                 }
             })
         }
@@ -428,12 +428,12 @@ pub fn test_multi_sites_with_gc(
                     2 => {
                         info_span!("FastSnapshot", from = i, to = j).in_scope(|| {
                             b_doc
-                                .import(&a_doc.export(loro::ExportMode::Snapshot))
+                                .import(&a_doc.export(loro::ExportMode::Snapshot).unwrap())
                                 .unwrap();
                         });
                         info_span!("FastSnapshot", from = j, to = i).in_scope(|| {
                             a_doc
-                                .import(&b_doc.export(loro::ExportMode::Snapshot))
+                                .import(&b_doc.export(loro::ExportMode::Snapshot).unwrap())
                                 .unwrap();
                         });
                     }

diff --git a/crates/fuzz/tests/compatibility.rs b/crates/fuzz/tests/compatibility.rs
@@ -54,7 +54,7 @@ fn updates_with_commit_message_can_be_imported_to_016() {
     }
 
     let doc3 = loro::LoroDoc::new();
-    doc3.import(&doc1.export(loro::ExportMode::Snapshot))
+    doc3.import(&doc1.export(loro::ExportMode::Snapshot).unwrap())
         .unwrap();
     let change_from_2 = doc3.get_change(ID::new(doc2.peer_id(), 0)).unwrap();
     assert_eq!(change_from_2.len, 3);

diff --git a/crates/loro-common/src/error.rs b/crates/loro-common/src/error.rs
@@ -114,20 +114,28 @@ pub enum LoroTreeError {
 pub enum LoroEncodeError {
     #[error("The frontiers are not found in this doc: {0}")]
     FrontiersNotFound(String),
+    #[error("Trimmed snapshot incompatible with old snapshot format. Use new snapshot format or avoid trimmed snapshots for storage.")]
+    TrimmedSnapshotIncompatibleWithOldFormat,
 }
 
 #[cfg(feature = "wasm")]
 pub mod wasm {
     use wasm_bindgen::JsValue;
 
-    use crate::LoroError;
+    use crate::{LoroEncodeError, LoroError};
 
     impl From<LoroError> for JsValue {
         fn from(value: LoroError) -> Self {
             JsValue::from_str(&value.to_string())
         }
     }
 
+    impl From<LoroEncodeError> for JsValue {
+        fn from(value: LoroEncodeError) -> Self {
+            JsValue::from_str(&value.to_string())
+        }
+    }
+
     impl From<JsValue> for LoroError {
         fn from(v: JsValue) -> Self {
             Self::JsError(