Add RPC benchmarks

pouya-eghbali · Nov 19, 2024 · 3e7aabc · 3e7aabc
1 parent 4a0964d
commit 3e7aabc
Show file tree

Hide file tree

Showing 9 changed files with 418 additions and 5 deletions.
diff --git a/benchmark.log b/benchmark.log
@@ -0,0 +1,53 @@
+ubuntu@ip-172-30-1-158:~/sia$ yarn benchmark
+Running serialization benchmark...
+┌─────────┬─────────────────┬───────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ 'JSON'          │ '10547111.90 ± 0.07%' │ '10470255.00'       │ '95 ± 0.06%'               │ '96'                      │ 5689    │
+│ 1       │ 'Sializer'      │ '1660856.80 ± 0.02%'  │ '1656032.00 ± 2.00' │ '602 ± 0.02%'              │ '604'                     │ 36126   │
+│ 2       │ 'Sializer (v1)' │ '3276995.83 ± 0.22%'  │ '3198344.00 ± 1.00' │ '308 ± 0.09%'              │ '313'                     │ 18310   │
+│ 3       │ 'CBOR-X'        │ '3676483.99 ± 0.15%'  │ '3606842.00 ± 1.00' │ '274 ± 0.09%'              │ '277'                     │ 16320   │
+│ 4       │ 'MsgPackr'      │ '3779219.59 ± 0.02%'  │ '3770572.00'        │ '265 ± 0.02%'              │ '265'                     │ 15877   │
+└─────────┴─────────────────┴───────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Running deserialization benchmark...
+┌─────────┬─────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ 'JSON'          │ '2488537.97 ± 0.19%' │ '2383928.00'        │ '406 ± 0.11%'              │ '419'                     │ 24111   │
+│ 1       │ 'Sializer'      │ '2013603.23 ± 0.14%' │ '1893356.00 ± 1.00' │ '503 ± 0.12%'              │ '528'                     │ 29798   │
+│ 2       │ 'Sializer (v1)' │ '3281259.84 ± 0.09%' │ '3171218.50 ± 1.50' │ '306 ± 0.09%'              │ '315'                     │ 18286   │
+│ 3       │ 'CBOR-X'        │ '3655012.13 ± 0.11%' │ '3511568.50 ± 2.50' │ '275 ± 0.09%'              │ '285'                     │ 16416   │
+│ 4       │ 'MsgPackr'      │ '3942333.41 ± 0.13%' │ '3757221.00 ± 2.00' │ '255 ± 0.11%'              │ '266'                     │ 15220   │
+└─────────┴─────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Sia file size: 777815
+Sia v1 file size: 932824
+JSON file size: 1402814
+MsgPackr file size: 1130312
+CBOR-X file size: 1138361
+ubuntu@ip-172-30-1-158:~/sia$ yarn benchmark
+Running serialization benchmark...
+┌─────────┬─────────────────┬───────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns)  │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼───────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ 'JSON'          │ '10909254.38 ± 0.11%' │ '10867102.00'       │ '92 ± 0.10%'               │ '92'                      │ 5500    │
+│ 1       │ 'Sializer'      │ '1629795.48 ± 0.02%'  │ '1628848.00'        │ '614 ± 0.02%'              │ '614'                     │ 36815   │
+│ 2       │ 'Sializer (v1)' │ '3187076.26 ± 0.16%'  │ '3128592.00'        │ '316 ± 0.08%'              │ '320'                     │ 18827   │
+│ 3       │ 'CBOR-X'        │ '3568670.79 ± 0.15%'  │ '3496599.00'        │ '282 ± 0.09%'              │ '286'                     │ 16813   │
+│ 4       │ 'MsgPackr'      │ '3766867.28 ± 0.20%'  │ '3700340.00'        │ '268 ± 0.10%'              │ '270'                     │ 15929   │
+└─────────┴─────────────────┴───────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Running deserialization benchmark...
+┌─────────┬─────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐
+│ (index) │ Task name       │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │
+├─────────┼─────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤
+│ 0       │ 'JSON'          │ '2508390.18 ± 0.21%' │ '2403629.00'        │ '403 ± 0.11%'              │ '416'                     │ 23920   │
+│ 1       │ 'Sializer'      │ '2019126.49 ± 0.15%' │ '1892181.00'        │ '502 ± 0.12%'              │ '528'                     │ 29716   │
+│ 2       │ 'Sializer (v1)' │ '3285009.18 ± 0.10%' │ '3167650.00'        │ '306 ± 0.09%'              │ '316'                     │ 18265   │
+│ 3       │ 'CBOR-X'        │ '3644615.38 ± 0.09%' │ '3503953.00'        │ '275 ± 0.09%'              │ '285'                     │ 16463   │
+│ 4       │ 'MsgPackr'      │ '4011836.49 ± 0.12%' │ '3826482.00 ± 5.00' │ '251 ± 0.11%'              │ '261'                     │ 14956   │
+└─────────┴─────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘
+Sia file size: 776898
+Sia v1 file size: 931879
+JSON file size: 1401897
+MsgPackr file size: 1129244
+CBOR-X file size: 1137086
+ubuntu@ip-172-30-1-158:~/sia$ 
diff --git a/package.json b/package.json
@@ -8,19 +8,25 @@
     "build": "tsc",
     "lint": "tslint -p tsconfig.json",
     "prepublishOnly": "yarn build",
-    "benchmark": "yarn build && yarn node dist/benchmark/index.js"
+    "benchmark": "yarn build && yarn node dist/benchmark/index.js",
+    "benchmark:ws:server": "yarn build && yarn node dist/benchmark/ws/simple/server.js",
+    "benchmark:ws": "yarn build && yarn node dist/benchmark/ws/simple/index.js",
+    "benchmark:ws:server:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/server.js",
+    "benchmark:ws:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/index.js"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
     "@faker-js/faker": "^9.2.0",
     "@types/node": "^22.9.0",
+    "@types/ws": "^8",
     "cbor-x": "^1.6.0",
     "msgpackr": "^1.11.2",
     "sializer": "0",
     "tinybench": "^3.0.6",
-    "typescript": "^5.4.3"
+    "typescript": "^5.4.3",
+    "ws": "^8.18.0"
   },
   "dependencies": {
     "utfz-lib": "^0.2.0"

diff --git a/src/benchmark/tests/common.ts b/src/benchmark/tests/common.ts
@@ -16,6 +16,10 @@ export const fiveUsers = faker.helpers.multiple(createRandomUser, {
   count: 5,
 });
 
+export const fiveHundredUsers = faker.helpers.multiple(createRandomUser, {
+  count: 500,
+});
+
 export const fiveThousandUsers = faker.helpers.multiple(createRandomUser, {
   count: 5_000,
 });
diff --git a/src/benchmark/ws/heavy/index.ts b/src/benchmark/ws/heavy/index.ts
@@ -0,0 +1,108 @@
+import { Bench } from "tinybench";
+import WebSocket from "ws";
+import { Sia } from "../../../index.js";
+import { pack, unpack } from "msgpackr";
+import { decode, encode } from "cbor-x";
+import { fiveHundredUsers } from "../../tests/common.js";
+
+const sia = new Sia();
+
+const rpcRequest = {
+  method: "batchCalculateUserAges",
+  params: fiveHundredUsers,
+};
+
+export const payloads = {
+  sia: () =>
+    sia
+      .seek(0)
+      .addAscii(rpcRequest.method)
+      .addArray16(rpcRequest.params, (sia, user) => {
+        sia
+          .addAscii(user.userId)
+          .addAscii(user.username)
+          .addAscii(user.email)
+          .addAscii(user.avatar)
+          .addAscii(user.password)
+          .addInt64(user.birthdate.getTime())
+          .addInt64(user.registeredAt.getTime());
+      })
+      .toUint8ArrayReference(),
+  json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))),
+  cbor: () => new Uint8Array(encode(rpcRequest)),
+  msgpack: () => new Uint8Array(pack(rpcRequest)),
+};
+
+const clients = {
+  sia: new WebSocket("ws://localhost:8080"),
+  cbor: new WebSocket("ws://localhost:8081"),
+  msgpack: new WebSocket("ws://localhost:8082"),
+  json: new WebSocket("ws://localhost:8083"),
+};
+
+const callbacks = {
+  sia: (data: Buffer) => {
+    return new Sia(new Uint8Array(data)).readArray16((sia) => {
+      const userId = sia.readAscii();
+      const age = sia.readUInt8();
+      return { userId, age };
+    });
+  },
+  cbor: (data: Buffer) => decode(data),
+  msgpack: (data: Buffer) => unpack(data),
+  json: (data: Buffer) => JSON.parse(data.toString()),
+};
+
+console.log("Waiting for connections...");
+await new Promise((resolve) => setTimeout(resolve, 15 * 1000));
+
+const bench = new Bench({ name: "RPC", time: 10 * 1000 });
+
+const makeRpcCall = async (
+  ws: WebSocket,
+  ondata: (data: Buffer) => void,
+  payload: Uint8Array
+) =>
+  new Promise((resolve) => {
+    ws.send(payload, { binary: true });
+    const done = (data: Buffer) => {
+      ws.off("message", done);
+      ondata(data);
+      resolve(null);
+    };
+    ws.on("message", done);
+  });
+
+bench
+  .add(
+    "JSON",
+    async () => await makeRpcCall(clients.json, callbacks.json, payloads.json())
+  )
+  .addEventListener("complete", () => clients.json.close());
+
+bench
+  .add(
+    "Sia",
+    async () => await makeRpcCall(clients.sia, callbacks.sia, payloads.sia())
+  )
+  .addEventListener("complete", () => clients.sia.close());
+
+bench
+  .add(
+    "CBOR",
+    async () => await makeRpcCall(clients.cbor, callbacks.cbor, payloads.cbor())
+  )
+  .addEventListener("complete", () => clients.cbor.close());
+
+bench
+  .add(
+    "MsgPack",
+    async () =>
+      await makeRpcCall(clients.msgpack, callbacks.msgpack, payloads.msgpack())
+  )
+  .addEventListener("complete", () => clients.msgpack.close());
+
+console.log(`Running ${bench.name} benchmark...`);
+await bench.run();
+
+console.table(bench.table());
diff --git a/src/benchmark/ws/heavy/server.ts b/src/benchmark/ws/heavy/server.ts
@@ -0,0 +1,81 @@
+import { WebSocketServer } from "ws";
+import { Sia } from "../../../index.js";
+import { pack, unpack } from "msgpackr";
+import { encode, decode } from "cbor-x";
+
+const sia = new Sia();
+
+type User = { userId: string; birthdate: Date };
+
+const getUserAge = (user: User) => ({
+  userId: user.userId,
+  age: new Date().getFullYear() - user.birthdate.getFullYear(),
+});
+
+const BASE_PORT = 8080;
+
+console.log("Starting Sia WS server on port", BASE_PORT);
+const siaWss = new WebSocketServer({ port: BASE_PORT + 0 });
+siaWss.on("connection", function connection(ws) {
+  ws.on("error", console.error);
+  ws.on("message", (data) => {
+    // Read and skip method name
+    sia.setContent(data as Buffer).readAscii();
+    const users = sia.readArray16((sia: Sia) => {
+      const userId = sia.readAscii();
+      sia.readAscii(); // username
+      sia.readAscii(); // email
+      sia.readAscii(); // avatar
+      sia.readAscii(); // password
+      const birthdate = new Date(sia.readInt64());
+      sia.readInt64(); // registeredAt
+      return { userId, birthdate };
+    });
+    const ages = users.map(getUserAge);
+    const payload = sia
+      .seek(0)
+      .addArray16(ages, (sia: Sia, age) => {
+        sia.addAscii(age.userId).addUInt8(age.age);
+      })
+      .toUint8ArrayReference();
+    ws.send(payload, { binary: true });
+  });
+});
+
+console.log("Starting CBOR WS server on port", BASE_PORT + 1);
+const cborWss = new WebSocketServer({ port: BASE_PORT + 1 });
+cborWss.on("connection", function connection(ws) {
+  ws.on("error", console.error);
+  ws.on("message", (data) => {
+    const users = decode(data as Buffer).params;
+    const ages = users.map(getUserAge);
+    const payload = encode(ages);
+    ws.send(payload, { binary: true });
+  });
+});
+
+console.log("Starting MsgPack WS server on port", BASE_PORT + 2);
+const msgpackWss = new WebSocketServer({ port: BASE_PORT + 2 });
+msgpackWss.on("connection", function connection(ws) {
+  ws.on("error", console.error);
+  ws.on("message", (data) => {
+    const users = unpack(data as Buffer).params;
+    const ages = users.map(getUserAge);
+    const payload = pack(ages);
+    ws.send(payload, { binary: true });
+  });
+});
+
+console.log("Starting JSON WS server on port", BASE_PORT + 3);
+const jsonWss = new WebSocketServer({ port: BASE_PORT + 3 });
+jsonWss.on("connection", function connection(ws) {
+  ws.on("error", console.error);
+  ws.on("message", (data) => {
+    const users = JSON.parse(data.toString()).params;
+    const ages = users
+      .map((user: User) => ({ ...user, birthdate: new Date(user.birthdate) }))
+      .map(getUserAge);
+    const payload = new Uint8Array(Buffer.from(JSON.stringify(ages)));
+    ws.send(payload);
+  });
+});
diff --git a/src/benchmark/ws/simple/index.ts b/src/benchmark/ws/simple/index.ts
@@ -0,0 +1,71 @@
+import { Bench } from "tinybench";
+import WebSocket from "ws";
+import { Sia } from "../../../index.js";
+import { pack } from "msgpackr";
+import { encode } from "cbor-x";
+
+const address = "0x1234567890123456789012345678901234567890";
+const sia = new Sia();
+
+const rpcRequest = {
+  method: "getBalance",
+  params: [address],
+};
+
+export const payloads = {
+  sia: () =>
+    sia
+      .seek(0)
+      .addAscii(rpcRequest.method)
+      .addAscii(rpcRequest.params[0])
+      .toUint8ArrayReference(),
+  json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))),
+  cbor: () => new Uint8Array(encode(rpcRequest)),
+  msgpack: () => new Uint8Array(pack(rpcRequest)),
+};
+
+const clients = {
+  sia: new WebSocket("ws://localhost:8080"),
+  cbor: new WebSocket("ws://localhost:8081"),
+  msgpack: new WebSocket("ws://localhost:8082"),
+  json: new WebSocket("ws://localhost:8083"),
+};
+
+console.log("Waiting for connections...");
+await new Promise((resolve) => setTimeout(resolve, 15 * 1000));
+
+const bench = new Bench({ name: "RPC", time: 10 * 1000 });
+
+const makeRpcCall = async (ws: WebSocket, payload: Uint8Array) =>
+  new Promise((resolve) => {
+    ws.send(payload, { binary: true });
+    const done = () => {
+      ws.off("message", done);
+      resolve(null);
+    };
+    ws.on("message", done);
+  });
+
+bench
+  .add("JSON", async () => await makeRpcCall(clients.json, payloads.json()))
+  .addEventListener("complete", () => clients.json.close());
+
+bench
+  .add("Sia", async () => await makeRpcCall(clients.sia, payloads.sia()))
+  .addEventListener("complete", () => clients.sia.close());
+
+bench
+  .add("CBOR", async () => await makeRpcCall(clients.cbor, payloads.cbor()))
+  .addEventListener("complete", () => clients.cbor.close());
+
+bench
+  .add(
+    "MsgPack",
+    async () => await makeRpcCall(clients.msgpack, payloads.msgpack())
+  )
+  .addEventListener("complete", () => clients.msgpack.close());
+
+console.log(`Running ${bench.name} benchmark...`);
+await bench.run();
+
+console.table(bench.table());