Use Custom Rust Targets for Shared Wasm Libraries (#4683)

* Patch Rust for shared Wasm libraries * Get rid of `musl` * Further simplify Wasm linking * Update benchmark results * No longer need `wasm-ld` patch * Update test result * Add distinction for Linux Ocaml build * Revert "No longer need `wasm-ld` patch" This reverts commit 3d0f34e. * Revert "Update test result" This reverts commit 8fc5c08. * Use patched wasm-ld during Rust build * Table offset must be at least 1 for Rust RTS * Adjust linker test result * Use bulk memory operations * Do not need `wasm-lld` patch with shared targets * Adjust expected test result * Adjust nix build script * Fix nix build script
dfinity · Oct 8, 2024 · aa445dd · aa445dd
1 parent a6d1305
commit aa445dd
Show file tree

Hide file tree

Showing 23 changed files with 220 additions and 360 deletions.
diff --git a/.github/workflows/niv-updater-rare.yml b/.github/workflows/niv-updater-rare.yml
@@ -23,7 +23,7 @@ jobs:
         uses: knl/niv-updater-action@v13
         with:
           # might be too noisy
-          whitelist: 'ic-ref,musl-wasi,ic-wasm'
+          whitelist: 'ic-ref,ic-wasm'
           labels: |
             automerge-squash
           keep_updating: true

diff --git a/.github/workflows/niv-updater-trial.yml b/.github/workflows/niv-updater-trial.yml
@@ -28,7 +28,7 @@ jobs:
       - name: niv-updater-action
         uses: knl/niv-updater-action@v13
         with:
-          whitelist: 'nixpkgs,musl-wasi,ic'
+          whitelist: 'nixpkgs,ic'
           labels: |
             autoclose
           keep_updating: true

diff --git a/.github/workflows/niv-updater.yml b/.github/workflows/niv-updater.yml
@@ -23,7 +23,7 @@ jobs:
         uses: knl/niv-updater-action@v13
         with:
           # might be too noisy
-          blacklist: 'nixpkgs,ic-ref,musl-wasi,ic'
+          blacklist: 'nixpkgs,ic-ref,ic'
           labels: |
             automerge-squash
           keep_updating: true

diff --git a/default.nix b/default.nix
@@ -23,63 +23,10 @@ let ic-ref-run =
 let haskellPackages = nixpkgs.haskellPackages.override {
       overrides = import nix/haskell-packages.nix nixpkgs subpath;
     }; in
-let llvm_sources = nixpkgs.fetchFromGitHub {
-      owner = "llvm";
-      repo = "llvm-project";
-      rev = "llvmorg-18.1.8";
-      sha256 = "sha256-iiZKMRo/WxJaBXct9GdAcAT3cz9d9pnAcO1mmR6oPNE=";
-    }; in
-let patched-wasm-ld = stdenv.mkDerivation {
-    name = "wasm-ld";
-    src = llvm_sources;
-    nativeBuildInputs = with nixpkgs; [ cmake ninja ];
-    buildInputs = with nixpkgs; [ llvmPackages_18.libllvm libxml2 ];
-
-    patchPhase = ''
-      patch lld/wasm/Relocations.cpp  << EOF
-@@ -104,9 +104,15 @@ void scanRelocations(InputChunk *chunk) {
-     case R_WASM_TABLE_INDEX_SLEB64:
-     case R_WASM_TABLE_INDEX_REL_SLEB:
-     case R_WASM_TABLE_INDEX_REL_SLEB64:
--      if (requiresGOTAccess(sym))
--        break;
-+      if (!sym->isDefined()) {
-+        error(toString(file) + ": relocation " + relocTypeToString(reloc.Type) +
-+              " cannot be used against an undefined symbol \`" + toString(*sym) +
-+              "\`");
-+      }
-       out.elemSec->addEntry(cast<FunctionSymbol>(sym));
-+      if (requiresGOTAccess(sym)) {
-+        addGOTEntry(sym);
-+      }
-       break;
-     case R_WASM_GLOBAL_INDEX_LEB:
-     case R_WASM_GLOBAL_INDEX_I32:
-EOF
-    cd lld
-    '';
-
-    outputs = [ "out" ];
-  }; in
-# Selectively build llvm binary tools.
-# Exclude wasm-ld, as this is patched separately.
-let llvm_bintools = stdenv.mkDerivation {
-    name = "llvm_bintools";
-    src = llvm_sources;
-    nativeBuildInputs = with nixpkgs; [ cmake ninja python3 ];
-    buildInputs = with nixpkgs; [ llvmPackages_18.libllvm libxml2 ];
-
-    preConfigure = ''
-      cd llvm
-    '';
-
-    outputs = [ "out" ];
-  }; in
 let
   rtsBuildInputs = with nixpkgs; [
     llvmPackages_18.clang
-    llvm_bintools
-    patched-wasm-ld
+    llvmPackages_18.bintools
     rustc-nightly
     cargo-nightly
     wasmtime
@@ -193,15 +140,6 @@ let ocaml_exe = name: bin: rts:
         $out/bin/* --help >/dev/null
       '';
     };
-
-  musl-wasi-sysroot = stdenv.mkDerivation {
-    name = "musl-wasi-sysroot";
-    src = nixpkgs.sources.musl-wasi;
-    phases = [ "unpackPhase" "installPhase" ];
-    installPhase = ''
-      make SYSROOT="$out" include_dirs
-    '';
-  };
 in
 
 rec {
@@ -295,8 +233,6 @@ rec {
         
         ${llvmEnv}
         export TOMMATHSRC=${nixpkgs.sources.libtommath}
-        export MUSLSRC=${nixpkgs.sources.musl-wasi}/libc-top-half/musl
-        export MUSL_WASI_SYSROOT=${musl-wasi-sysroot}
       '';
 
       doCheck = true;
@@ -383,7 +319,7 @@ rec {
 
     # extra deps for test/ld
     ldTestDeps =
-      with nixpkgs; [ patched-wasm-ld llvmPackages_18.clang ];
+      with nixpkgs; [ llvmPackages_18.lld llvmPackages_18.clang ];
 
     testDerivation = args:
       stdenv.mkDerivation (testDerivationArgs // args);
@@ -906,8 +842,6 @@ EOF
     '';
     ESM=nixpkgs.sources.esm;
     TOMMATHSRC = nixpkgs.sources.libtommath;
-    MUSLSRC = "${nixpkgs.sources.musl-wasi}/libc-top-half/musl";
-    MUSL_WASI_SYSROOT = musl-wasi-sysroot;
     LOCALE_ARCHIVE = nixpkgs.lib.optionalString stdenv.isLinux "${nixpkgs.glibcLocales}/lib/locale/locale-archive";
     MOTOKO_BASE = base-src;
     CANDID_TESTS = "${nixpkgs.sources.candid}/test";

diff --git a/nix/default.nix b/nix/default.nix
@@ -78,7 +78,7 @@ let
                 };
               };
 
-              # No testing of atdgen, as it pulls in python stuff, tricky on musl
+              # No testing of atdgen, as it pulls in python stuff
               atdgen = super.ocamlPackages.atdgen.overrideAttrs { doCheck = false; };
             };
           }
@@ -95,9 +95,7 @@ let
         in rec {
           rustc-nightly = rust-channel.rust.override {
             targets = [
-               "wasm32-unknown-emscripten"
                "wasm32-wasi"
-               "i686-unknown-linux-gnu"
             ];
             extensions = ["rust-src"];
           };

diff --git a/nix/sources.json b/nix/sources.json
@@ -94,19 +94,6 @@
         "url": "https://github.com/kritzcreek/motoko-matchers/archive/cb838c192df3328ff9ae172e2dc7338cf55e74bf.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "musl-wasi": {
-        "branch": "main",
-        "builtin": false,
-        "description": "WASI libc implementation for WebAssembly",
-        "homepage": "https://wasi.dev",
-        "owner": "WebAssembly",
-        "repo": "wasi-libc",
-        "rev": "c5264e2bbe532994d06b039005f2af91bedcc1a6",
-        "sha256": "1skw2jqzaidr2zbzbjw32b36xvqniwf4if8cr4kbpp8vjvh7j2lr",
-        "type": "tarball",
-        "url": "https://github.com/WebAssembly/wasi-libc/archive/c5264e2bbe532994d06b039005f2af91bedcc1a6.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "nixpkgs": {
         "branch": "release-24.05",
         "description": "Nix Packages collection & NixOS",