Enhanced Orthogonal Persistence (64-Bit without Graph Copy)

default.nix

@@ -20,6 +20,10 @@ let ic-ref-run =
       cp ${ic-hs-pkgs.ic-hs}/bin/ic-ref-run $out/bin
   ''; in
 
+let
+  nixos-unstable = import nixpkgs.sources.nixpkgs-unstable {};
+in
+
 let haskellPackages = nixpkgs.haskellPackages.override {
       overrides = import nix/haskell-packages.nix nixpkgs subpath;
     }; in
@@ -35,6 +39,7 @@ let
     wasmtime
     rust-bindgen
     python3
+    nixos-unstable.emscripten
   ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
     libiconv
   ];
@@ -193,7 +198,7 @@ rec {
         name = "motoko-rts-deps";
         src = subpath ./rts;
         sourceRoot = "rts/motoko-rts-tests";
-        sha256 = "sha256-jN5nx5UNBHlYKnC0kk90h6mWPUNrqPS7Wln2TixbGgA=";
+        sha256 = "sha256-YEw2AFi15JVKP7owsPDoBT3qSegOO90FRn2qoUBAICw=";
         copyLockfile = true;
       };
 
@@ -241,6 +246,7 @@ rec {
           "directory" = "$(stripHash ${allDeps})"
         __END__
 
+
         ${llvmEnv}
         export TOMMATHSRC=${nixpkgs.sources.libtommath}
         export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
@@ -415,7 +421,7 @@ rec {
       '';
     };
 
-    # wasm-profiler is not compatible with passive data segments
+    # wasm-profiler is not compatible with passive data segments and memory64
     # profiling-graphs = testDerivation {
     #  src = test_src "perf";
     #  buildInputs =
@@ -481,7 +487,7 @@ rec {
       perf       = perf_subdir "perf"       [ moc nixpkgs.drun ];
       bench      = perf_subdir "bench"      [ moc nixpkgs.drun ic-wasm ];
       # viper      = test_subdir "viper"      [ moc nixpkgs.which nixpkgs.openjdk nixpkgs.z3 ];
-      # TODO: profiling-graph is excluded because the underlying partity_wasm is deprecated and does not support passive data segments.
+      # TODO: profiling-graph is excluded because the underlying partity_wasm is deprecated and does not support passive data segments and memory64.
       inherit qc lsp unit candid coverage;
     }) // { recurseForDerivations = true; };
 
@@ -526,7 +532,7 @@ rec {
         doInstallCheck = true;
         test = ./test + "/test-${n}.js";
         installCheckPhase = ''
-          NODE_PATH=$out/bin node $test
+          NODE_PATH=$out/bin node --experimental-wasm-memory64 $test
         '';
       };
     in
@@ -749,8 +755,9 @@ EOF
       check-grammar
       check-error-codes
     ] ++
-    builtins.attrValues tests ++
-    builtins.attrValues js;
+    builtins.attrValues tests
+    ++ builtins.attrValues js
+    ;
   };
 
   viperServer = nixpkgs.fetchurl {

design/OrthogonalPersistence.md

@@ -1,29 +1,31 @@
-# Orthogonal Persistence (Stable Heap)
+# Orthogonal Persistence (64-Bit, Scalable Upgrades)
 
-This realizes the vision of keeping the canister main memory persistent even across upgrades and thus allows scalable upgrades.
-Canister upgrades do no longer involve serialization and deserialization to and from secondary stable memory.
+This implements the vision of **enhanced orthogonal persistence** in Motoko that combines:
+* **Stable heap**: Persisting the program main memory across canister upgrades.
+* **64-bit heap**: Extending the main memory to 64-bit for large-scaled persistence.
 
-## Purpose
-* **Instantenous upgrades**: New program versions simply resume on the existing main memory and have access to the memory-compatible data.
+As a result, the use of secondary storage (explicit stable memory, dedicated stable data structures, DB-like storage abstractions) will no longer be necessary: Motoko developers can directly work on their normal object-oriented program structures that are automatically persisted and retained across program version changes.
+
+## Advantages
+Compared to the existing orthogonal persistence in Motoko, this design offers:
+* **Instantaneous upgrades**: New program versions immediately resume from the existing main memory and have access to the memory-compatible data.
 * **Scalable upgrades**: The upgrade mechanism scales with larger heaps and in contrast to serialization, does not hit IC instruction limits.
 
-## Broader Vision
-In the longer term, this approach aims to enable **true orthogonal persistence** that is simple, flexible, efficient, and scalable.
-While this version implements the runtime support for 32-bit memory, this could be leveraged to 64-bit persistent main memory in future.
-As a result, the use of secondary storage (explicit stable memory, dedicated stable data structures, DB-like storage abstractions) will no longer be necessary: 
-Motoko developers could directly work on their normal object-oriented program structures that are automatically persisted and retained across program version changes.
-With 64-bit main memory, large-scaled orthogonal persistence would be enabled, supported by the incremental GC that is designed to also scale in 64-bit.
+Compared to the explicit use of stable memory, this design improves:
+* **Simplicity**: Developers do not need to deal with explicit stable memory.
+* **Performance**: No copying to and from the separate stable memory is necessary.
 
 ## Design
 The stable heap is based on the following main properties:
 * Extension of the IC to retain main memory on upgrades.
+* Supporting 64-bit main memory on the IC.
 * A long-term memory layout that is invariant to new compiled program versions.
 * A fast memory compatibility check performed on each canister upgrade.
 * Incremental garbage collection using a partitioned heap.
 
 ### IC Extension
-As a prerequisite for the stable heap support, the IC runtime support has to be extended in order not to erase the main memory on upgrades.
-This is realized in a specific IC PR (https://github.com/luc-blaeser/ic/tree/luc/stable-heap-on-release) that retains the main memory even on upgrades, similar to normal canister message execution. 
+The necessary IC extensions are implemented in a separate PR: https://github.com/dfinity/ic/pull/143
+This PR is based on these extensions.
 
 ### Memory Layout
 In a co-design between the compiler and the runtime system, the main memory is arranged in the following structure, invariant of the compiled program version:
@@ -65,10 +67,9 @@ This compatibility check serves as an additional safety measure on top of the DF
 ### Garbage Collection
 The implementation focuses on the incremental GC and abandons the other GCs because the GCs use different memory layouts. For example, the incremental GC uses a partitioned heap with objects carrying a forwarding pointer.
 
-The incremental GC is chosen because it is designed to scale on large heaps and the stable heap design also aims to increase scalability. Moreover, it is suited to scale on 64-bit memory in future.
+The incremental GC is chosen because it is designed to scale on large heaps and the stable heap design also aims to increase scalability.
 
-The garbage collection state needs to be persisted and retained across upgrades. 
-This is because the GC may not yet be completed at the time of an upgrade, such that object forwarding is still in use. The partition table is stored as part of the GC state.
+The garbage collection state needs to be persisted and retained across upgrades. This is because the GC may not yet be completed at the time of an upgrade, such that object forwarding is still in use. The partition table is stored as part of the GC state.
 
 The garbage collector uses two kinds of roots:
 * Persistent roots: These refer to root objects that need to survive canister upgrades.
@@ -82,14 +83,13 @@ The transient roots are referenced by the Wasm data segments and comprise:
 * All canister variables of the current version, including flexible variables.
 
 ### Main Actor
-On an upgrade, the main actor is recreated and existing stable variables are recovered from the persistent root.
-The remaining actor variables, the flexible fields as well as new stable variables, are (re)initialized.
+On an upgrade, the main actor is recreated and existing stable variables are recovered from the persistent root. The remaining actor variables, the flexible fields as well as new stable variables, are (re)initialized.
 As a result, the GC can collect unreachable flexible objects of previous canister versions. 
 Unused stable variables of former versions can also be reclaimed by the GC.
 
 ### No Static Heap
 The static heap is abandoned and former static objects need to be allocated in the dynamic heap.
-This is because these objects may also need to survive upgrades and must not be not overwritten by new data segments. 
+This is because these objects may also need to survive upgrades and must not be not overwritten by new data segments. The incremental GC also operates on these objects, meaning that forwarding pointer resolution is also necessary for these objects. 
 
 The incremental GC also operates on these objects, meaning that forwarding pointer resolution is also necessary for these objects.
 
@@ -114,18 +114,14 @@ This is acceptable because the RTS only uses small size for data segments (e.g.
 As an optimization, the top-level `null` pointer is represented as a constant sentinel value pointing to the last unallocated Wasm page. This allows fast null tests without involving forwarding pointer resolution of potential non-null comparand pointers.
 
 ### Migration Path
-When migrating from the old serialization-based stabilization to the new stable heap, the old data is deserialized one last time from stable memory and then placed in the new stable heap layout.
-Once operating on the stable heap, the system prevents downgrade attempts to the old serialization-based persistence.
+When migrating from the old serialization-based stabilization to the new stable heap, the old data is deserialized one last time from stable memory and then placed in the new stable heap layout. Once operating on the stable heap, the system prevents downgrade attempts to the old serialization-based persistence. Assuming that the persistent memory layout needs to be changed in the future, a future version of the compiler/runtime system can perform such upgrade by serializing/deserializing the heap to stable memory.
 
 ### Old Stable Memory
 The old stable memory remains equally accessible as secondary memory with the new support.
 
 ## Current Limitations
-* Freeing old object fields: While new program versions can drop object fields, the runtime system should also delete the redundant fields of persistent objects of previous program versions. This could be realized during garbage collection when objects are copied. For this purpose, the runtime system may maintain a set of field hashes in use and consult this table during garbage collection.
-* Bounded Rust call stack size: The Rust call stack size needs to be bounded and can no longer be configured by the user.
-* The Wasm profiler (only used for the flamegraphs) is no longer applicable because the underlying `parity-wasm` crate lacks full support of passive data segments. A re-implementation of the profiler would be needed.
-
-## Related PRs
-
-* IC with stable main memory support: https://github.com/luc-blaeser/ic/tree/luc/stable-heap-on-release
-* Wasm64 Support for Motoko: https://github.com/dfinity/motoko/pull/4136
+* The incremental GC only allows 64 GB. Transitioning to a dynamic partition table would be necessary to go beyond this limit. This is to be be implemented in a separate PR.
+* The floating point display format differs in Wasm64 for special values, e.g. `nan` becomes `NaN`. There is currently no support for hexadecimal floating point text formatting.
+* Workaround for Rust needed to build PIC (position-independent code) libraries. Explicit use of `emscripten` via LLVM IR. 
+* `ic-wasm` would need to be extended to support memory64 and passive data segments. The Wasm optimizations in `test/bench` are thus currently deactivated.
+* The Wasm profiler is no longer applicable because the underlying `parity-wasm` crate seems deprecated. A re-implementation of the profiler would be needed.

nix/sources.json

@@ -70,15 +70,15 @@
         "version": "v1.2.0"
     },
     "motoko-base": {
-        "branch": "next-moc",
+        "branch": "pull/589/head",
         "description": "The Motoko base library",
-        "homepage": null,
+        "homepage": "",
         "owner": "dfinity",
         "repo": "motoko-base",
-        "rev": "dfabfc3201950263a0057f5d1a78d2ede5c87478",
-        "sha256": "1wd9rcmhg82h08l3gz3gmn2xpgalvf679w7yz2g35yzbg2b5g2fd",
+        "rev": "1aa2b8638245d7541e646f5edd326b2384ee92b4",
+        "sha256": "128v62p0wizncmp7d3zb3gxnagd11gnr29msqjb90k7f4klxpaiv",
         "type": "tarball",
-        "url": "https://github.com/dfinity/motoko-base/archive/dfabfc3201950263a0057f5d1a78d2ede5c87478.tar.gz",
+        "url": "https://github.com/dfinity/motoko-base/archive/1aa2b8638245d7541e646f5edd326b2384ee92b4.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "motoko-matchers": {
@@ -133,6 +133,18 @@
         "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/1ca9ee7192f973fd67b0988bdd77b8c11ae245a6.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
+    "nixpkgs-unstable": {
+        "branch": "master",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9d2864289e65d0dc841123ae86bc7b3f77b6d12d",
+        "sha256": "0f3bcphzrjm1hs81mhpfx1hqxfnbvhhv5hv82wi1h3f735i279ja",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/9d2864289e65d0dc841123ae86bc7b3f77b6d12d.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "ocaml-vlq": {
         "branch": "v0.2.0",
         "builtin": false,

rts/.gitignore

@@ -1,3 +1,2 @@
-mo-rts.wasm
-mo-rts-debug.wasm
+mo*.wasm
 _build