From 3e7aabcd338e4564b818a0ee67fd0110a16e9b1c Mon Sep 17 00:00:00 2001 From: Pouya Eghbali Date: Tue, 19 Nov 2024 20:56:00 +0700 Subject: [PATCH] Add RPC benchmarks --- benchmark.log | 53 +++++++++++++++ package.json | 10 ++- src/benchmark/tests/common.ts | 4 ++ src/benchmark/ws/heavy/index.ts | 108 ++++++++++++++++++++++++++++++ src/benchmark/ws/heavy/server.ts | 81 ++++++++++++++++++++++ src/benchmark/ws/simple/index.ts | 71 ++++++++++++++++++++ src/benchmark/ws/simple/server.ts | 62 +++++++++++++++++ src/index.ts | 6 +- yarn.lock | 28 +++++++- 9 files changed, 418 insertions(+), 5 deletions(-) create mode 100644 benchmark.log create mode 100644 src/benchmark/ws/heavy/index.ts create mode 100644 src/benchmark/ws/heavy/server.ts create mode 100644 src/benchmark/ws/simple/index.ts create mode 100644 src/benchmark/ws/simple/server.ts diff --git a/benchmark.log b/benchmark.log new file mode 100644 index 0000000..e3c3b8b --- /dev/null +++ b/benchmark.log @@ -0,0 +1,53 @@ +ubuntu@ip-172-30-1-158:~/sia$ yarn benchmark +Running serialization benchmark... +┌─────────┬─────────────────┬───────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐ +│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │ +├─────────┼─────────────────┼───────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤ +│ 0 │ 'JSON' │ '10547111.90 ± 0.07%' │ '10470255.00' │ '95 ± 0.06%' │ '96' │ 5689 │ +│ 1 │ 'Sializer' │ '1660856.80 ± 0.02%' │ '1656032.00 ± 2.00' │ '602 ± 0.02%' │ '604' │ 36126 │ +│ 2 │ 'Sializer (v1)' │ '3276995.83 ± 0.22%' │ '3198344.00 ± 1.00' │ '308 ± 0.09%' │ '313' │ 18310 │ +│ 3 │ 'CBOR-X' │ '3676483.99 ± 0.15%' │ '3606842.00 ± 1.00' │ '274 ± 0.09%' │ '277' │ 16320 │ +│ 4 │ 'MsgPackr' │ '3779219.59 ± 0.02%' │ '3770572.00' │ '265 ± 0.02%' │ '265' │ 15877 │ +└─────────┴─────────────────┴───────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘ +Running deserialization benchmark... +┌─────────┬─────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐ +│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │ +├─────────┼─────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤ +│ 0 │ 'JSON' │ '2488537.97 ± 0.19%' │ '2383928.00' │ '406 ± 0.11%' │ '419' │ 24111 │ +│ 1 │ 'Sializer' │ '2013603.23 ± 0.14%' │ '1893356.00 ± 1.00' │ '503 ± 0.12%' │ '528' │ 29798 │ +│ 2 │ 'Sializer (v1)' │ '3281259.84 ± 0.09%' │ '3171218.50 ± 1.50' │ '306 ± 0.09%' │ '315' │ 18286 │ +│ 3 │ 'CBOR-X' │ '3655012.13 ± 0.11%' │ '3511568.50 ± 2.50' │ '275 ± 0.09%' │ '285' │ 16416 │ +│ 4 │ 'MsgPackr' │ '3942333.41 ± 0.13%' │ '3757221.00 ± 2.00' │ '255 ± 0.11%' │ '266' │ 15220 │ +└─────────┴─────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘ +Sia file size: 777815 +Sia v1 file size: 932824 +JSON file size: 1402814 +MsgPackr file size: 1130312 +CBOR-X file size: 1138361 +ubuntu@ip-172-30-1-158:~/sia$ yarn benchmark +Running serialization benchmark... +┌─────────┬─────────────────┬───────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐ +│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │ +├─────────┼─────────────────┼───────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤ +│ 0 │ 'JSON' │ '10909254.38 ± 0.11%' │ '10867102.00' │ '92 ± 0.10%' │ '92' │ 5500 │ +│ 1 │ 'Sializer' │ '1629795.48 ± 0.02%' │ '1628848.00' │ '614 ± 0.02%' │ '614' │ 36815 │ +│ 2 │ 'Sializer (v1)' │ '3187076.26 ± 0.16%' │ '3128592.00' │ '316 ± 0.08%' │ '320' │ 18827 │ +│ 3 │ 'CBOR-X' │ '3568670.79 ± 0.15%' │ '3496599.00' │ '282 ± 0.09%' │ '286' │ 16813 │ +│ 4 │ 'MsgPackr' │ '3766867.28 ± 0.20%' │ '3700340.00' │ '268 ± 0.10%' │ '270' │ 15929 │ +└─────────┴─────────────────┴───────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘ +Running deserialization benchmark... +┌─────────┬─────────────────┬──────────────────────┬─────────────────────┬────────────────────────────┬───────────────────────────┬─────────┐ +│ (index) │ Task name │ Latency average (ns) │ Latency median (ns) │ Throughput average (ops/s) │ Throughput median (ops/s) │ Samples │ +├─────────┼─────────────────┼──────────────────────┼─────────────────────┼────────────────────────────┼───────────────────────────┼─────────┤ +│ 0 │ 'JSON' │ '2508390.18 ± 0.21%' │ '2403629.00' │ '403 ± 0.11%' │ '416' │ 23920 │ +│ 1 │ 'Sializer' │ '2019126.49 ± 0.15%' │ '1892181.00' │ '502 ± 0.12%' │ '528' │ 29716 │ +│ 2 │ 'Sializer (v1)' │ '3285009.18 ± 0.10%' │ '3167650.00' │ '306 ± 0.09%' │ '316' │ 18265 │ +│ 3 │ 'CBOR-X' │ '3644615.38 ± 0.09%' │ '3503953.00' │ '275 ± 0.09%' │ '285' │ 16463 │ +│ 4 │ 'MsgPackr' │ '4011836.49 ± 0.12%' │ '3826482.00 ± 5.00' │ '251 ± 0.11%' │ '261' │ 14956 │ +└─────────┴─────────────────┴──────────────────────┴─────────────────────┴────────────────────────────┴───────────────────────────┴─────────┘ +Sia file size: 776898 +Sia v1 file size: 931879 +JSON file size: 1401897 +MsgPackr file size: 1129244 +CBOR-X file size: 1137086 +ubuntu@ip-172-30-1-158:~/sia$ \ No newline at end of file diff --git a/package.json b/package.json index d2f2cdb..a804078 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,11 @@ "build": "tsc", "lint": "tslint -p tsconfig.json", "prepublishOnly": "yarn build", - "benchmark": "yarn build && yarn node dist/benchmark/index.js" + "benchmark": "yarn build && yarn node dist/benchmark/index.js", + "benchmark:ws:server": "yarn build && yarn node dist/benchmark/ws/simple/server.js", + "benchmark:ws": "yarn build && yarn node dist/benchmark/ws/simple/index.js", + "benchmark:ws:server:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/server.js", + "benchmark:ws:heavy": "yarn build && yarn node dist/benchmark/ws/heavy/index.js" }, "keywords": [], "author": "", @@ -16,11 +20,13 @@ "devDependencies": { "@faker-js/faker": "^9.2.0", "@types/node": "^22.9.0", + "@types/ws": "^8", "cbor-x": "^1.6.0", "msgpackr": "^1.11.2", "sializer": "0", "tinybench": "^3.0.6", - "typescript": "^5.4.3" + "typescript": "^5.4.3", + "ws": "^8.18.0" }, "dependencies": { "utfz-lib": "^0.2.0" diff --git a/src/benchmark/tests/common.ts b/src/benchmark/tests/common.ts index 2d102b3..6c8c0ed 100644 --- a/src/benchmark/tests/common.ts +++ b/src/benchmark/tests/common.ts @@ -16,6 +16,10 @@ export const fiveUsers = faker.helpers.multiple(createRandomUser, { count: 5, }); +export const fiveHundredUsers = faker.helpers.multiple(createRandomUser, { + count: 500, +}); + export const fiveThousandUsers = faker.helpers.multiple(createRandomUser, { count: 5_000, }); diff --git a/src/benchmark/ws/heavy/index.ts b/src/benchmark/ws/heavy/index.ts new file mode 100644 index 0000000..f938270 --- /dev/null +++ b/src/benchmark/ws/heavy/index.ts @@ -0,0 +1,108 @@ +import { Bench } from "tinybench"; +import WebSocket from "ws"; +import { Sia } from "../../../index.js"; +import { pack, unpack } from "msgpackr"; +import { decode, encode } from "cbor-x"; +import { fiveHundredUsers } from "../../tests/common.js"; + +const sia = new Sia(); + +const rpcRequest = { + method: "batchCalculateUserAges", + params: fiveHundredUsers, +}; + +export const payloads = { + sia: () => + sia + .seek(0) + .addAscii(rpcRequest.method) + .addArray16(rpcRequest.params, (sia, user) => { + sia + .addAscii(user.userId) + .addAscii(user.username) + .addAscii(user.email) + .addAscii(user.avatar) + .addAscii(user.password) + .addInt64(user.birthdate.getTime()) + .addInt64(user.registeredAt.getTime()); + }) + .toUint8ArrayReference(), + json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))), + cbor: () => new Uint8Array(encode(rpcRequest)), + msgpack: () => new Uint8Array(pack(rpcRequest)), +}; + +const clients = { + sia: new WebSocket("ws://localhost:8080"), + cbor: new WebSocket("ws://localhost:8081"), + msgpack: new WebSocket("ws://localhost:8082"), + json: new WebSocket("ws://localhost:8083"), +}; + +const callbacks = { + sia: (data: Buffer) => { + return new Sia(new Uint8Array(data)).readArray16((sia) => { + const userId = sia.readAscii(); + const age = sia.readUInt8(); + return { userId, age }; + }); + }, + cbor: (data: Buffer) => decode(data), + msgpack: (data: Buffer) => unpack(data), + json: (data: Buffer) => JSON.parse(data.toString()), +}; + +console.log("Waiting for connections..."); +await new Promise((resolve) => setTimeout(resolve, 15 * 1000)); + +const bench = new Bench({ name: "RPC", time: 10 * 1000 }); + +const makeRpcCall = async ( + ws: WebSocket, + ondata: (data: Buffer) => void, + payload: Uint8Array +) => + new Promise((resolve) => { + ws.send(payload, { binary: true }); + const done = (data: Buffer) => { + ws.off("message", done); + ondata(data); + resolve(null); + }; + ws.on("message", done); + }); + +bench + .add( + "JSON", + async () => await makeRpcCall(clients.json, callbacks.json, payloads.json()) + ) + .addEventListener("complete", () => clients.json.close()); + +bench + .add( + "Sia", + async () => await makeRpcCall(clients.sia, callbacks.sia, payloads.sia()) + ) + .addEventListener("complete", () => clients.sia.close()); + +bench + .add( + "CBOR", + async () => await makeRpcCall(clients.cbor, callbacks.cbor, payloads.cbor()) + ) + .addEventListener("complete", () => clients.cbor.close()); + +bench + .add( + "MsgPack", + async () => + await makeRpcCall(clients.msgpack, callbacks.msgpack, payloads.msgpack()) + ) + .addEventListener("complete", () => clients.msgpack.close()); + +console.log(`Running ${bench.name} benchmark...`); +await bench.run(); + +console.table(bench.table()); diff --git a/src/benchmark/ws/heavy/server.ts b/src/benchmark/ws/heavy/server.ts new file mode 100644 index 0000000..d65c569 --- /dev/null +++ b/src/benchmark/ws/heavy/server.ts @@ -0,0 +1,81 @@ +import { WebSocketServer } from "ws"; +import { Sia } from "../../../index.js"; +import { pack, unpack } from "msgpackr"; +import { encode, decode } from "cbor-x"; + +const sia = new Sia(); + +type User = { userId: string; birthdate: Date }; + +const getUserAge = (user: User) => ({ + userId: user.userId, + age: new Date().getFullYear() - user.birthdate.getFullYear(), +}); + +const BASE_PORT = 8080; + +console.log("Starting Sia WS server on port", BASE_PORT); +const siaWss = new WebSocketServer({ port: BASE_PORT + 0 }); +siaWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + // Read and skip method name + sia.setContent(data as Buffer).readAscii(); + const users = sia.readArray16((sia: Sia) => { + const userId = sia.readAscii(); + sia.readAscii(); // username + sia.readAscii(); // email + sia.readAscii(); // avatar + sia.readAscii(); // password + const birthdate = new Date(sia.readInt64()); + sia.readInt64(); // registeredAt + return { userId, birthdate }; + }); + const ages = users.map(getUserAge); + const payload = sia + .seek(0) + .addArray16(ages, (sia: Sia, age) => { + sia.addAscii(age.userId).addUInt8(age.age); + }) + .toUint8ArrayReference(); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting CBOR WS server on port", BASE_PORT + 1); +const cborWss = new WebSocketServer({ port: BASE_PORT + 1 }); +cborWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const users = decode(data as Buffer).params; + const ages = users.map(getUserAge); + const payload = encode(ages); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting MsgPack WS server on port", BASE_PORT + 2); +const msgpackWss = new WebSocketServer({ port: BASE_PORT + 2 }); +msgpackWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const users = unpack(data as Buffer).params; + const ages = users.map(getUserAge); + const payload = pack(ages); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting JSON WS server on port", BASE_PORT + 3); +const jsonWss = new WebSocketServer({ port: BASE_PORT + 3 }); +jsonWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const users = JSON.parse(data.toString()).params; + const ages = users + .map((user: User) => ({ ...user, birthdate: new Date(user.birthdate) })) + .map(getUserAge); + const payload = new Uint8Array(Buffer.from(JSON.stringify(ages))); + ws.send(payload); + }); +}); diff --git a/src/benchmark/ws/simple/index.ts b/src/benchmark/ws/simple/index.ts new file mode 100644 index 0000000..12b4b11 --- /dev/null +++ b/src/benchmark/ws/simple/index.ts @@ -0,0 +1,71 @@ +import { Bench } from "tinybench"; +import WebSocket from "ws"; +import { Sia } from "../../../index.js"; +import { pack } from "msgpackr"; +import { encode } from "cbor-x"; + +const address = "0x1234567890123456789012345678901234567890"; +const sia = new Sia(); + +const rpcRequest = { + method: "getBalance", + params: [address], +}; + +export const payloads = { + sia: () => + sia + .seek(0) + .addAscii(rpcRequest.method) + .addAscii(rpcRequest.params[0]) + .toUint8ArrayReference(), + json: () => new Uint8Array(Buffer.from(JSON.stringify(rpcRequest))), + cbor: () => new Uint8Array(encode(rpcRequest)), + msgpack: () => new Uint8Array(pack(rpcRequest)), +}; + +const clients = { + sia: new WebSocket("ws://localhost:8080"), + cbor: new WebSocket("ws://localhost:8081"), + msgpack: new WebSocket("ws://localhost:8082"), + json: new WebSocket("ws://localhost:8083"), +}; + +console.log("Waiting for connections..."); +await new Promise((resolve) => setTimeout(resolve, 15 * 1000)); + +const bench = new Bench({ name: "RPC", time: 10 * 1000 }); + +const makeRpcCall = async (ws: WebSocket, payload: Uint8Array) => + new Promise((resolve) => { + ws.send(payload, { binary: true }); + const done = () => { + ws.off("message", done); + resolve(null); + }; + ws.on("message", done); + }); + +bench + .add("JSON", async () => await makeRpcCall(clients.json, payloads.json())) + .addEventListener("complete", () => clients.json.close()); + +bench + .add("Sia", async () => await makeRpcCall(clients.sia, payloads.sia())) + .addEventListener("complete", () => clients.sia.close()); + +bench + .add("CBOR", async () => await makeRpcCall(clients.cbor, payloads.cbor())) + .addEventListener("complete", () => clients.cbor.close()); + +bench + .add( + "MsgPack", + async () => await makeRpcCall(clients.msgpack, payloads.msgpack()) + ) + .addEventListener("complete", () => clients.msgpack.close()); + +console.log(`Running ${bench.name} benchmark...`); +await bench.run(); + +console.table(bench.table()); diff --git a/src/benchmark/ws/simple/server.ts b/src/benchmark/ws/simple/server.ts new file mode 100644 index 0000000..4be66c0 --- /dev/null +++ b/src/benchmark/ws/simple/server.ts @@ -0,0 +1,62 @@ +import { WebSocketServer } from "ws"; +import { Sia } from "../../../index.js"; +import { pack, unpack } from "msgpackr"; +import { encode, decode } from "cbor-x"; + +const address = "0x1234567890123456789012345678901234567890"; +const map = new Map([[address, Number.MAX_SAFE_INTEGER]]); + +const getAccountBalance = (address: string) => map.get(address) || 0; +const sia = new Sia(); + +const BASE_PORT = 8080; + +console.log("Starting Sia WS server on port", BASE_PORT); +const siaWss = new WebSocketServer({ port: BASE_PORT + 0 }); +siaWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + // Read and skip method name + sia.setContent(data as Buffer).readAscii(); + const address = sia.readAscii(); + const balance = getAccountBalance(address); + const payload = sia.seek(0).addInt64(balance).toUint8ArrayReference(); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting CBOR WS server on port", BASE_PORT + 1); +const cborWss = new WebSocketServer({ port: BASE_PORT + 1 }); +cborWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const address = decode(data as Buffer).params[0]; + const balance = getAccountBalance(address); + const payload = encode({ balance }); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting MsgPack WS server on port", BASE_PORT + 2); +const msgpackWss = new WebSocketServer({ port: BASE_PORT + 2 }); +msgpackWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const address = unpack(data as Buffer).params[0]; + const balance = getAccountBalance(address); + const payload = pack({ balance }); + ws.send(payload, { binary: true }); + }); +}); + +console.log("Starting JSON WS server on port", BASE_PORT + 3); +const jsonWss = new WebSocketServer({ port: BASE_PORT + 3 }); +jsonWss.on("connection", function connection(ws) { + ws.on("error", console.error); + ws.on("message", (data) => { + const address = JSON.parse(data.toString()).params[0]; + const balance = getAccountBalance(address); + const payload = new Uint8Array(Buffer.from(JSON.stringify({ balance }))); + ws.send(payload); + }); +}); diff --git a/src/index.ts b/src/index.ts index 37c1a5a..dba61d6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,9 +22,11 @@ export class Sia extends Buffer { return this; } - setContent(content: Uint8Array): Sia { - this.content = content; + setContent(uint8Array: Uint8Array): Sia { + this.size = uint8Array.length; + this.content = uint8Array; this.offset = 0; + this.dataView = new DataView(uint8Array.buffer); return this; } diff --git a/yarn.lock b/yarn.lock index 4443e27..cbcacb4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -139,7 +139,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^22.9.0": +"@types/node@npm:*, @types/node@npm:^22.9.0": version: 22.9.0 resolution: "@types/node@npm:22.9.0" dependencies: @@ -148,6 +148,15 @@ __metadata: languageName: node linkType: hard +"@types/ws@npm:^8": + version: 8.5.13 + resolution: "@types/ws@npm:8.5.13" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/a5430aa479bde588e69cb9175518d72f9338b6999e3b2ae16fc03d3bdcff8347e486dc031e4ed14601260463c07e1f9a0d7511dfc653712b047c439c680b0b34 + languageName: node + linkType: hard + "abbrev@npm:^2.0.0": version: 2.0.0 resolution: "abbrev@npm:2.0.0" @@ -925,12 +934,14 @@ __metadata: dependencies: "@faker-js/faker": "npm:^9.2.0" "@types/node": "npm:^22.9.0" + "@types/ws": "npm:^8" cbor-x: "npm:^1.6.0" msgpackr: "npm:^1.11.2" sializer: "npm:0" tinybench: "npm:^3.0.6" typescript: "npm:^5.4.3" utfz-lib: "npm:^0.2.0" + ws: "npm:^8.18.0" languageName: unknown linkType: soft @@ -1142,6 +1153,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:^8.18.0": + version: 8.18.0 + resolution: "ws@npm:8.18.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 10c0/25eb33aff17edcb90721ed6b0eb250976328533ad3cd1a28a274bd263682e7296a6591ff1436d6cbc50fa67463158b062f9d1122013b361cec99a05f84680e06 + languageName: node + linkType: hard + "yallist@npm:^4.0.0": version: 4.0.0 resolution: "yallist@npm:4.0.0"