From def5c4e20081f76cdc585b10feb43c7087f6cd4a Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Fri, 26 Apr 2024 17:41:26 +0200 Subject: [PATCH 01/10] feat: implement websockets --- scripts/global.js | 22 +--- scripts/network.js | 305 ++++++++++++++++++++------------------------ scripts/settings.js | 8 +- scripts/wallet.js | 19 ++- 4 files changed, 152 insertions(+), 202 deletions(-) diff --git a/scripts/global.js b/scripts/global.js index 4bad0b2c9..6005a2995 100644 --- a/scripts/global.js +++ b/scripts/global.js @@ -245,14 +245,11 @@ export async function start() { subscribeToNetworkEvents(); // Make sure we know the correct number of blocks - await refreshChainData(); + blockCount = (await getNetwork().getChainInfo())['bestHeight']; // If allowed by settings: submit a simple 'hit' (app load) to Labs Analytics getNetwork().submitAnalytics('hit'); setInterval(() => { - // Refresh blockchain data - refreshChainData(); - // Fetch the PIVX prices refreshPriceDisplay(); }, 15000); @@ -1647,23 +1644,6 @@ export async function createProposal() { } } -export async function refreshChainData() { - const cNet = getNetwork(); - // If in offline mode: don't sync ANY data or connect to the internet - if (!cNet.enabled) - return console.warn( - 'Offline mode active: For your security, the wallet will avoid ALL internet requests.' - ); - - // Fetch block count - const newBlockCount = await cNet.getBlockCount(); - if (newBlockCount !== blockCount) { - blockCount = newBlockCount; - if (!wallet.isLoaded()) return; - getEventEmitter().emit('new-block', blockCount); - } -} - // A safety mechanism enabled if the user attempts to leave without encrypting/saving their keys export const beforeUnloadListener = (evt) => { evt.preventDefault(); diff --git a/scripts/network.js b/scripts/network.js index 9a2ab9c26..fc08c8224 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -1,4 +1,3 @@ -import { cChainParams } from './chain_params.js'; import { createAlert } from './misc.js'; import { sleep } from './utils.js'; import { getEventEmitter } from './event_bus.js'; @@ -102,20 +101,120 @@ export class Network { } } -/** - * - */ export class ExplorerNetwork extends Network { /** * @param {string} strUrl - Url pointing to the blockbook explorer */ - constructor(strUrl, wallet) { - super(wallet); + constructor(strUrl) { + super(); + // ensure backward compatibility + if (strUrl.startsWith('http')) { + strUrl = strUrl.replace('http', 'ws'); + } + if (!strUrl.endsWith('/websocket')) { + strUrl += '/websocket'; + } /** * @type{string} * @public */ this.strUrl = strUrl; + this.cachedResults = []; + this.subscriptions = []; + this.ID = 0; + this.ws = new WebSocket(strUrl); + this.ws.onopen = function (e) { + console.log('socket connected', e); + }; + this.ws.onclose = function (e) { + console.log('socket closed', e); + }; + this.ws.onmessage = function (e) { + const resp = JSON.parse(e.data); + // Is this a subscription? + const f = _network.subscriptions[resp.id]; + if (f !== undefined) { + f(resp.data); + } + // If it isn't cache the result + _network.cachedResults[resp.id] = resp.data; + }; + } + close() { + this.ws.close(); + } + async init() { + for (let i = 0; i < 100; i++) { + if (this.ws.readyState === WebSocket.OPEN) { + break; + } + await sleep(100); + } + if (this.ws.readyState !== WebSocket.OPEN) { + throw new Error('Cannot connect to websocket!'); + } + this.subscribeNewBlock(); + } + + send(method, params) { + const id = this.ID.toString(); + const req = { + id, + method, + params, + }; + this.ID++; + this.ws.send(JSON.stringify(req)); + return id; + } + async sendAndWaitForAnswer(method, params) { + let attempt = 0; + while (attempt <= 10) { + const id = this.send(method, params); + for (let i = 0; i < 100; i++) { + const res = this.cachedResults[id]; + if (res !== undefined) { + delete this.cachedResults[id]; + if (res.error) { + console.log('Failed attempt: ', attempt); + await sleep(1000); + break; + } + return res; + } + await sleep(100); + } + attempt += 1; + } + } + subscribe(method, params, callback) { + const id = this.ID.toString(); + this.subscriptions[id] = callback; + const req = { + id, + method, + params, + }; + this.ws.send(JSON.stringify(req)); + this.ID++; + return id; + } + subscribeNewBlock() { + this.subscribe('subscribeNewBlock', {}, function (result) { + if (result['height'] !== undefined) { + getEventEmitter().emit('new-block', result['height']); + } + }); + } + async getAccountInfo(descriptor, page, pageSize, from, details = 'txs') { + const params = { + descriptor, + details, + page, + pageSize, + from, + }; + return await this.sendAndWaitForAnswer('getAccountInfo', params); } error() { @@ -131,77 +230,17 @@ export class ExplorerNetwork extends Network { * @param {boolean} skipCoinstake - if true coinstake tx will be skipped * @returns {Promise} the block fetched from explorer */ - async getBlock(blockHeight, skipCoinstake = false) { - try { - const block = await this.safeFetchFromExplorer( - `/api/v2/block/${blockHeight}` - ); - const newTxs = []; - // This is bad. We're making so many requests - // This is a quick fix to try to be compliant with the blockbook - // API, and not the PIVX extension. - // In the Blockbook API /block doesn't have any chain specific information - // Like hex, shield info or what not. - // We could change /getshieldblocks to /getshieldtxs? - // In addition, always skip the coinbase transaction and in case the coinstake one - // TODO: once v6.0 and shield stake is activated we might need to change this optimization - for (const tx of block.txs.slice(skipCoinstake ? 2 : 1)) { - const r = await fetch( - `${this.strUrl}/api/v2/tx-specific/${tx.txid}` - ); - if (!r.ok) throw new Error('failed'); - const newTx = await r.json(); - newTxs.push(newTx); - } - block.txs = newTxs; - return block; - } catch (e) { - this.error(); - throw e; - } - } - - async getBlockCount() { + async getBlock(blockNumber) { try { - const { backend } = await ( - await retryWrapper(fetchBlockbook, `/api/v2/api`) - ).json(); - - return backend.blocks; + return await this.sendAndWaitForAnswer('getBlock', { + id: blockNumber.toString(), + }); } catch (e) { this.error(); throw e; } } - /** - * Sometimes blockbook might return internal error, in this case this function will sleep for some times and retry - * @param {string} strCommand - The specific Blockbook api to call - * @param {number} sleepTime - How many milliseconds sleep between two calls. Default value is 20000ms - * @returns {Promise} Explorer result in json - */ - async safeFetchFromExplorer(strCommand, sleepTime = 20000) { - let trials = 0; - const maxTrials = 6; - while (trials < maxTrials) { - trials += 1; - const res = await retryWrapper(fetchBlockbook, strCommand); - if (!res.ok) { - if (debug) { - console.log( - 'Blockbook internal error! sleeping for ' + - sleepTime + - ' seconds' - ); - } - await sleep(sleepTime); - continue; - } - return await res.json(); - } - throw new Error('Cannot safe fetch from explorer!'); - } - /** * //TODO: do not take the wallet as parameter but instead something weaker like a public key or address? * Must be called only for initial wallet sync @@ -218,14 +257,11 @@ export class ExplorerNetwork extends Network { if (debug) { console.time('getLatestTxsTimer'); } - // Form the API call using our wallet information - const strKey = wallet.getKeyToExport(); - const strRoot = `/api/v2/${ - wallet.isHD() ? 'xpub/' : 'address/' - }${strKey}`; - const strCoreParams = `?details=txs&from=${nStartHeight}`; - const probePage = await this.safeFetchFromExplorer( - `${strRoot + strCoreParams}&pageSize=1` + const probePage = await this.getAccountInfo( + wallet.getKeyToExport(), + 1, + 1, + nStartHeight ); const txNumber = probePage.txs - wallet.getTransactions().length; // Compute the total pages and iterate through them until we've synced everything @@ -241,8 +277,11 @@ export class ExplorerNetwork extends Network { ); // Fetch this page of transactions - const iPage = await this.safeFetchFromExplorer( - `${strRoot + strCoreParams}&page=${i}` + const iPage = await this.getAccountInfo( + wallet.getKeyToExport(), + i, + 10000, + nStartHeight ); // Update the internal mempool if there's new transactions @@ -281,17 +320,9 @@ export class ExplorerNetwork extends Network { * @returns {Promise>} Resolves when it has finished fetching UTXOs */ async getUTXOs(strAddress) { - try { - let publicKey = strAddress; - // Fetch UTXOs for the key - const arrUTXOs = await ( - await retryWrapper(fetchBlockbook, `/api/v2/utxo/${publicKey}`) - ).json(); - return arrUTXOs; - } catch (e) { - console.error(e); - this.error(); - } + return await this.sendAndWaitForAnswer('getAccountUtxo', { + descriptor: strAddress, + }); } /** @@ -300,19 +331,14 @@ export class ExplorerNetwork extends Network { * @returns {Promise} - A JSON class of aggregated XPUB info */ async getXPubInfo(strXPUB) { - return await ( - await retryWrapper(fetchBlockbook, `/api/v2/xpub/${strXPUB}`) - ).json(); + return await this.getAccountInfo(strXPUB, 1, 1, 0, 'tokens'); } async sendTransaction(hex) { try { - const data = await ( - await retryWrapper(fetchBlockbook, '/api/v2/sendtx/', { - method: 'post', - body: hex, - }) - ).json(); + const data = await this.sendAndWaitForAnswer('sendTransaction', { + hex, + }); // Throw and catch if the data is not a TXID if (!data.result || data.result.length !== 64) throw data; @@ -327,8 +353,17 @@ export class ExplorerNetwork extends Network { } async getTxInfo(txHash) { - const req = await retryWrapper(fetchBlockbook, `/api/v2/tx/${txHash}`); - return await req.json(); + return await this.sendAndWaitForAnswer('getTransaction', { + txid: txHash, + }); + } + + /** + * Get the blockchain info. + * This is used to get the blockchain height at start or when switching chain. + */ + async getChainInfo() { + return await this.sendAndWaitForAnswer('getInfo', {}); } /** @@ -377,8 +412,10 @@ let _network = null; * Sets the network in use by MPW. * @param {ExplorerNetwork} network - network to use */ -export function setNetwork(network) { +export async function setNetwork(network) { + _network?.close(); _network = network; + await _network.init(); } /** @@ -388,65 +425,3 @@ export function setNetwork(network) { export function getNetwork() { return _network; } - -/** - * A Fetch wrapper which uses the current Blockbook Network's base URL - * @param {string} api - The specific Blockbook api to call - * @param {RequestInit} options - The Fetch options - * @returns {Promise} - The unresolved Fetch promise - */ -export function fetchBlockbook(api, options) { - return fetch(_network.strUrl + api, options); -} - -/** - * A wrapper for Blockbook calls which can, in the event of an unresponsive explorer, - * seamlessly attempt the same call on multiple other explorers until success. - * @param {Function} func - The function to re-attempt with - * @param {...any} args - The arguments to pass to the function - */ -async function retryWrapper(func, ...args) { - // Track internal errors from the wrapper - let err; - - // If allowed by the user, Max Tries is ALL MPW-supported explorers, otherwise, restrict to only the current one. - let nMaxTries = cChainParams.current.Explorers.length; - let retries = 0; - - // The explorer index we started at - let nIndex = cChainParams.current.Explorers.findIndex( - (a) => a.url === getNetwork().strUrl - ); - - // Run the call until successful, or all attempts exhausted - while (retries < nMaxTries) { - try { - // Call the passed function with the arguments - const res = await func(...args); - - // If the endpoint is non-OK, assume it's an error - if (!res.ok) throw res; - - // Return the result if successful - return res; - } catch (error) { - err = error; - - // If allowed, switch explorers - if (!fAutoSwitch) throw err; - nIndex = (nIndex + 1) % cChainParams.current.Explorers.length; - const cNewExplorer = cChainParams.current.Explorers[nIndex]; - - // Set the explorer at Network-class level, then as a hacky workaround for the current callback; we - // ... adjust the internal URL to the new explorer. - getNetwork().strUrl = cNewExplorer.url; - setExplorer(cNewExplorer, true); - - // Bump the attempts, and re-try next loop - retries++; - } - } - - // Throw an error so the calling code knows the operation failed - throw err; -} diff --git a/scripts/settings.js b/scripts/settings.js index 4fd3b7fef..1e30b5f75 100644 --- a/scripts/settings.js +++ b/scripts/settings.js @@ -304,12 +304,8 @@ export async function setExplorer(explorer, fSilent = false) { cExplorer = explorer; // Enable networking + notify if allowed - if (getNetwork()) { - getNetwork().strUrl = cExplorer.url; - } else { - const network = new ExplorerNetwork(cExplorer.url, wallet); - setNetwork(network); - } + const network = new ExplorerNetwork(cExplorer.url); + await setNetwork(network); // Update the selector UI doms.domExplorerSelect.value = cExplorer.url; diff --git a/scripts/wallet.js b/scripts/wallet.js index 7f30a6d0d..1b330ac7f 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -738,7 +738,7 @@ export class Wallet { await startBatch( async (i) => { let block; - block = await cNet.getBlock(blockHeights[i], true); + block = await cNet.getBlock(blockHeights[i]); blocks[i] = block; // We need to process blocks monotically // When we get a block, start from the first unhandled @@ -751,17 +751,16 @@ export class Wallet { // Delete so we don't have to hold all blocks in memory // until we finish syncing delete blocks[j]; + getEventEmitter().emit( + 'shield-sync-status-update', + tr(translation.syncShieldProgress, [ + { current: handled }, + { total: blockHeights.length }, + ]), + false + ); syncing = false; } - - getEventEmitter().emit( - 'shield-sync-status-update', - tr(translation.syncShieldProgress, [ - { current: handled - 1 }, - { total: blockHeights.length }, - ]), - false - ); }, blockHeights.length, batchSize From 8070e6dcf774d914f49ee0c8328462d40b8e8809 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Tue, 7 May 2024 10:46:53 +0200 Subject: [PATCH 02/10] make the system more stable --- scripts/global.js | 10 ++++++- scripts/network.js | 72 ++++++++++++++++++++++++++++++++++----------- scripts/settings.js | 18 ++++++++++-- scripts/wallet.js | 2 +- 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/scripts/global.js b/scripts/global.js index 7da8510db..0d3baa97e 100644 --- a/scripts/global.js +++ b/scripts/global.js @@ -247,7 +247,7 @@ export async function start() { subscribeToNetworkEvents(); // Make sure we know the correct number of blocks - blockCount = (await getNetwork().getChainInfo())['bestHeight']; + await updateBlockCount(); // Load the price manager cOracle.load(); @@ -271,6 +271,14 @@ export async function start() { doms.domDashboard.click(); } +/** + * Updates the blockcount + * Must be called only on start and when toggling network + * @returns {Promise} + */ +export async function updateBlockCount() { + blockCount = await getNetwork().getBlockCount(); +} async function refreshPriceDisplay() { await cOracle.getPrice(strCurrency); getEventEmitter().emit('balance-update'); diff --git a/scripts/network.js b/scripts/network.js index 0d6972110..801d3d269 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -1,9 +1,11 @@ import { createAlert } from './misc.js'; import { + debugError, debugLog, debugTimerEnd, debugTimerStart, DebugTopics, + debugWarn, } from './debug.js'; import { sleep } from './utils.js'; import { getEventEmitter } from './event_bus.js'; @@ -11,12 +13,12 @@ import { STATS, cStatKeys, cAnalyticsLevel, - setExplorer, - fAutoSwitch, + setNextExplorer, } from './settings.js'; import { cNode } from './settings.js'; import { ALERTS, tr, translation } from './i18n.js'; import { Transaction } from './transaction.js'; +import * as net from 'net'; /** * @typedef {Object} XPUBAddress @@ -108,31 +110,39 @@ export class Network { export class ExplorerNetwork extends Network { /** - * @param {string} strUrl - Url pointing to the blockbook explorer + * @param {string} wsUrl - Url pointing to the blockbook explorer */ - constructor(strUrl) { + constructor(wsUrl) { super(); // ensure backward compatibility - if (strUrl.startsWith('http')) { - strUrl = strUrl.replace('http', 'ws'); + if (wsUrl.startsWith('http')) { + wsUrl = wsUrl.replace('http', 'ws'); } - if (!strUrl.endsWith('/websocket')) { - strUrl += '/websocket'; + if (!wsUrl.endsWith('/websocket')) { + wsUrl += '/websocket'; } /** * @type{string} * @public */ - this.strUrl = strUrl; + this.wsUrl = wsUrl; + this.closed = false; this.cachedResults = []; this.subscriptions = []; this.ID = 0; - this.ws = new WebSocket(strUrl); + this.ws = new WebSocket(wsUrl); this.ws.onopen = function (e) { - console.log('socket connected', e); + debugLog(DebugTopics.NET, 'websocket connected', e); }; this.ws.onclose = function (e) { - console.log('socket closed', e); + debugLog(DebugTopics.NET, 'websocket disconnected', e); + if (!e.wasClean) { + debugError( + DebugTopics.NET, + 'websocket unexpected close, trying to reconnect' + ); + setNextExplorer(); + } }; this.ws.onmessage = function (e) { const resp = JSON.parse(e.data); @@ -145,8 +155,15 @@ export class ExplorerNetwork extends Network { _network.cachedResults[resp.id] = resp.data; }; } + get strUrl() { + return this.wsUrl.replace('ws', 'http').replace('/websocket', ''); + } + close() { this.ws.close(); + this.cachedResults = []; + this.subscriptions = []; + this.closed = true; } async init() { for (let i = 0; i < 100; i++) { @@ -162,6 +179,9 @@ export class ExplorerNetwork extends Network { } send(method, params) { + if (this.closed) { + throw new Error('Trying to send with a closed explorer'); + } const id = this.ID.toString(); const req = { id, @@ -173,15 +193,15 @@ export class ExplorerNetwork extends Network { return id; } async sendAndWaitForAnswer(method, params) { - let attempt = 0; - while (attempt <= 10) { + let attempt = 1; + const maxAttempts = 5; + while (attempt <= maxAttempts) { const id = this.send(method, params); for (let i = 0; i < 100; i++) { const res = this.cachedResults[id]; if (res !== undefined) { delete this.cachedResults[id]; if (res.error) { - console.log('Failed attempt: ', attempt); await sleep(1000); break; } @@ -189,8 +209,16 @@ export class ExplorerNetwork extends Network { } await sleep(100); } + debugWarn( + DebugTopics.NET, + 'Failed send attempt for ' + method, + 'attempt ' + attempt + '/' + maxAttempts + ); attempt += 1; } + if (!this.closed) { + throw new Error('Failed to communicate with the explorer'); + } } subscribe(method, params, callback) { const id = this.ID.toString(); @@ -362,12 +390,21 @@ export class ExplorerNetwork extends Network { /** * Get the blockchain info. - * This is used to get the blockchain height at start or when switching chain. + * Internal function used to get the blockchain info. */ - async getChainInfo() { + async #getChainInfo() { return await this.sendAndWaitForAnswer('getInfo', {}); } + /** + * Returns the best known block height. + * Must be called only on start and when toggling mainnet/testnet + * @returns {Promise} + */ + async getBlockCount() { + return (await this.#getChainInfo())['bestHeight']; + } + /** * @return {Promise} The list of blocks which have at least one shield transaction */ @@ -415,6 +452,7 @@ let _network = null; * @param {ExplorerNetwork} network - network to use */ export async function setNetwork(network) { + debugLog(DebugTopics.NET, 'Connecting to new explorer', network.wsUrl); _network?.close(); _network = network; await _network.init(); diff --git a/scripts/settings.js b/scripts/settings.js index 5fef7d3a9..08c8ea9c9 100644 --- a/scripts/settings.js +++ b/scripts/settings.js @@ -3,7 +3,7 @@ import { updateLogOutButton, updateGovernanceTab, dashboard, - refreshChainData, + updateBlockCount, } from './global.js'; import { wallet, hasEncryptedWallet } from './wallet.js'; import { cChainParams } from './chain_params.js'; @@ -302,7 +302,7 @@ function subscribeToNetworkEvents() { } // --- Settings Functions -export async function setExplorer(explorer, fSilent = false) { +async function setExplorer(explorer, fSilent = false) { const database = await Database.getInstance(); database.setSettings({ explorer: explorer.url }); cExplorer = explorer; @@ -322,6 +322,18 @@ export async function setExplorer(explorer, fSilent = false) { ); } +export async function setNextExplorer() { + // The explorer index we started at + let nIndex = cChainParams.current.Explorers.findIndex( + (a) => a.url === getNetwork().strUrl + ); + if (fAutoSwitch) { + nIndex = (nIndex + 1) % cChainParams.current.Explorers.length; + } + const cNewExplorer = cChainParams.current.Explorers[nIndex]; + await setExplorer(cNewExplorer, true); +} + async function setNode(node, fSilent = false) { cNode = node; const database = await Database.getInstance(); @@ -558,7 +570,7 @@ export async function toggleTestnet() { doms.domTestnetToggler.checked = cChainParams.current.isTestnet; await start(); // Make sure we have the correct number of blocks before loading any wallet - await refreshChainData(); + await updateBlockCount(); getEventEmitter().emit('toggle-network'); await updateGovernanceTab(); } diff --git a/scripts/wallet.js b/scripts/wallet.js index 62f129448..68d2cd988 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -840,7 +840,7 @@ export class Wallet { for (const tx of block.txs) { const parsed = Transaction.fromHex(tx.hex); parsed.blockHeight = blockHeight; - parsed.blockTime = tx.blocktime; + parsed.blockTime = tx.blockTime; // Avoid wasting memory on txs that do not regard our wallet if (this.ownTransaction(parsed)) { await this.addTransaction(parsed); From 1ce65b6d4003631a30903f6b5321a0c82a6d9d50 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Tue, 7 May 2024 11:13:54 +0200 Subject: [PATCH 03/10] test: fix broken tests --- scripts/__mocks__/global.js | 15 ++++++++++++++- scripts/__mocks__/network.js | 11 ++++++++++- scripts/global.js | 2 +- tests/integration/wallet/sync.spec.js | 13 +++++++------ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/scripts/__mocks__/global.js b/scripts/__mocks__/global.js index 67b053520..dcc8c3a6e 100644 --- a/scripts/__mocks__/global.js +++ b/scripts/__mocks__/global.js @@ -1 +1,14 @@ -export const blockCount = 1504903; +import { getEventEmitter } from '../event_bus.js'; +import { vi } from 'vitest'; + +export let blockCount = 1504903; + +export const start = vi.fn(() => { + subscribeToNetworkEvents(); +}); + +const subscribeToNetworkEvents = vi.fn(() => { + getEventEmitter().on('new-block', (block) => { + blockCount = block; + }); +}); diff --git a/scripts/__mocks__/network.js b/scripts/__mocks__/network.js index bd393b50a..bca99b184 100644 --- a/scripts/__mocks__/network.js +++ b/scripts/__mocks__/network.js @@ -1,5 +1,7 @@ import { vi } from 'vitest'; import { Transaction } from '../transaction.js'; +import { getEventEmitter } from '../event_bus.js'; +import { sleep } from '../utils.js'; export const getNetwork = vi.fn(() => { return globalNetwork; @@ -106,6 +108,10 @@ class TestNetwork { this.#nextBlock.reset(); this.#nextBlock.blockHeight = this.#blockHeight + 1; } + mintAndEmit() { + this.mintBlock(); + getEventEmitter().emit('new-block', this.#blockHeight); + } } /** @@ -157,6 +163,9 @@ class TestTransaction { let globalNetwork = new TestNetwork(); -export function resetNetwork() { +export async function resetNetwork() { globalNetwork = new TestNetwork(); + // Update the global variable blockCount + getEventEmitter().emit('new-block', globalNetwork.getBlockCount()); + await sleep(100); } diff --git a/scripts/global.js b/scripts/global.js index 0d3baa97e..e0375aefa 100644 --- a/scripts/global.js +++ b/scripts/global.js @@ -292,7 +292,7 @@ function subscribeToNetworkEvents() { getEventEmitter().on('new-block', (block) => { debugLog(DebugTopics.GLOBAL, `New block detected! ${block}`); - + blockCount = block; // If it's open: update the Governance Dashboard if (doms.domGovTab.classList.contains('active')) { updateGovernanceTab(); diff --git a/tests/integration/wallet/sync.spec.js b/tests/integration/wallet/sync.spec.js index 9a5f4f7d0..3a283d1cb 100644 --- a/tests/integration/wallet/sync.spec.js +++ b/tests/integration/wallet/sync.spec.js @@ -9,10 +9,11 @@ import { getNetwork, resetNetwork, } from '../../../scripts/__mocks__/network.js'; -import { refreshChainData } from '../../../scripts/global.js'; import { sleep } from '../../../scripts/utils.js'; +import { start } from '../../../scripts/__mocks__/global.js'; vi.mock('../../../scripts/network.js'); +vi.mock('../../../scripts/global.js'); /** * @param{import('scripts/wallet').Wallet} wallet - wallet that will generate the transaction @@ -28,8 +29,7 @@ async function crateAndSendTransaction(wallet, address, value) { } async function mineAndSync() { - getNetwork().mintBlock(); - await refreshChainData(); + getNetwork().mintAndEmit(); // 500 milliseconds are enough time to make the wallets sync and handle the new blocks await sleep(500); } @@ -37,10 +37,11 @@ async function mineAndSync() { describe('Wallet sync tests', () => { let walletHD; let walletLegacy; + beforeAll(async () => { + await start(); + }); beforeEach(async () => { - resetNetwork(); - // Update the global variable blockCount - await refreshChainData(); + await resetNetwork(); walletHD = await setUpHDMainnetWallet(false); walletLegacy = await setUpLegacyMainnetWallet(); // Reset indexedDB before each test From 4fd267d58ea45409f31adfc83a588f2652230e10 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Tue, 7 May 2024 11:16:42 +0200 Subject: [PATCH 04/10] remove unused import --- scripts/network.js | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/network.js b/scripts/network.js index 801d3d269..4240688f8 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -18,7 +18,6 @@ import { import { cNode } from './settings.js'; import { ALERTS, tr, translation } from './i18n.js'; import { Transaction } from './transaction.js'; -import * as net from 'net'; /** * @typedef {Object} XPUBAddress From 1b536fcf0f5b6af1b112aee5090661f90c8f9353 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Wed, 8 May 2024 08:15:47 +0200 Subject: [PATCH 05/10] increase maxAttempts to 10 --- scripts/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/network.js b/scripts/network.js index 4240688f8..d0ee188b1 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -193,7 +193,7 @@ export class ExplorerNetwork extends Network { } async sendAndWaitForAnswer(method, params) { let attempt = 1; - const maxAttempts = 5; + const maxAttempts = 10; while (attempt <= maxAttempts) { const id = this.send(method, params); for (let i = 0; i < 100; i++) { From 7f87188ff081f8b24718be267532e732eb382f23 Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 8 May 2024 20:44:13 +0200 Subject: [PATCH 06/10] Increase max timeout to 600s and improve debug log --- scripts/network.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/network.js b/scripts/network.js index d0ee188b1..54f5ee762 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -194,24 +194,34 @@ export class ExplorerNetwork extends Network { async sendAndWaitForAnswer(method, params) { let attempt = 1; const maxAttempts = 10; + // milliseconds + const maxAwaitTime = 600 * 1000; + const frequency = 100; while (attempt <= maxAttempts) { + let receivedInvalidAnswer = false; const id = this.send(method, params); - for (let i = 0; i < 100; i++) { + for (let i = 0; i < Math.floor(maxAwaitTime / frequency); i++) { const res = this.cachedResults[id]; if (res !== undefined) { delete this.cachedResults[id]; if (res.error) { await sleep(1000); + receivedInvalidAnswer = true; break; } return res; } - await sleep(100); + await sleep(frequency); } debugWarn( DebugTopics.NET, 'Failed send attempt for ' + method, - 'attempt ' + attempt + '/' + maxAttempts + 'attempt ' + + attempt + + '/' + + maxAttempts + + ' received an invalid answer: ' + + receivedInvalidAnswer ); attempt += 1; } From 67a14099d5fbbe2b1b7046fc00055d311ae0961d Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 8 May 2024 22:13:32 +0200 Subject: [PATCH 07/10] pageSize = 1000 --- scripts/network.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/network.js b/scripts/network.js index 54f5ee762..f922d026d 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -320,7 +320,7 @@ export class ExplorerNetwork extends Network { const iPage = await this.getAccountInfo( wallet.getKeyToExport(), i, - 10000, + 1000, nStartHeight ); From 44fcdee20c7bc7f5f0dfa6b114eb621134aa0c38 Mon Sep 17 00:00:00 2001 From: ale Date: Wed, 8 May 2024 22:21:16 +0200 Subject: [PATCH 08/10] exit early if conn got closed --- scripts/network.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/network.js b/scripts/network.js index f922d026d..7de329c91 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -201,6 +201,9 @@ export class ExplorerNetwork extends Network { let receivedInvalidAnswer = false; const id = this.send(method, params); for (let i = 0; i < Math.floor(maxAwaitTime / frequency); i++) { + if (this.closed) { + break; + } const res = this.cachedResults[id]; if (res !== undefined) { delete this.cachedResults[id]; From b143346790f37d12df689ba9f66f82fe4c9ea309 Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Thu, 16 May 2024 23:07:48 +0200 Subject: [PATCH 09/10] websocket rework --- scripts/network.js | 144 +++++++++++++++++++++++++++----------------- scripts/settings.js | 3 +- 2 files changed, 89 insertions(+), 58 deletions(-) diff --git a/scripts/network.js b/scripts/network.js index 7de329c91..97c89c020 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -111,8 +111,17 @@ export class ExplorerNetwork extends Network { /** * @param {string} wsUrl - Url pointing to the blockbook explorer */ - constructor(wsUrl) { + constructor() { super(); + this.cachedResults = []; + this.subscriptions = []; + this.ID = 0; + } + async createWebSocketConnection(wsUrl) { + // Make sure the old connection is closed before opening a new one + if (this.ws) { + await this.reset(); + } // ensure backward compatibility if (wsUrl.startsWith('http')) { wsUrl = wsUrl.replace('http', 'ws'); @@ -120,15 +129,11 @@ export class ExplorerNetwork extends Network { if (!wsUrl.endsWith('/websocket')) { wsUrl += '/websocket'; } - /** - * @type{string} - * @public - */ this.wsUrl = wsUrl; - this.closed = false; - this.cachedResults = []; - this.subscriptions = []; - this.ID = 0; + this.openWebSocketConnection(wsUrl); + await this.initWebSocketConnection(); + } + openWebSocketConnection(wsUrl) { this.ws = new WebSocket(wsUrl); this.ws.onopen = function (e) { debugLog(DebugTopics.NET, 'websocket connected', e); @@ -140,10 +145,16 @@ export class ExplorerNetwork extends Network { DebugTopics.NET, 'websocket unexpected close, trying to reconnect' ); + // Connection closed somehow, try to reconnect setNextExplorer(); } }; this.ws.onmessage = function (e) { + // Return early if the websocket is not open + // In this way we avoid the case in which a closing websocket connection put values in the cache + if (this.readyState !== WebSocket.OPEN) { + return; + } const resp = JSON.parse(e.data); // Is this a subscription? const f = _network.subscriptions[resp.id]; @@ -154,43 +165,85 @@ export class ExplorerNetwork extends Network { _network.cachedResults[resp.id] = resp.data; }; } + async initWebSocketConnection() { + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); + await this.subscribeNewBlock(); + } get strUrl() { return this.wsUrl.replace('ws', 'http').replace('/websocket', ''); } - close() { - this.ws.close(); + async reset() { + if (this.ws.readyState == WebSocket.OPEN) { + this.ws.close(); + } + // Make sure websocket got closed before emptying arrays + await this.awaitWebSocketStatus( + WebSocket.CLOSED, + 'Websocket is not getting closed' + ); + // At this point messages of old websocket will not be received anymore + // and, it is safe to reset cachedResults and subscriptions this.cachedResults = []; this.subscriptions = []; - this.closed = true; } - async init() { + + async awaitWebSocketStatus(status, errMessage) { for (let i = 0; i < 100; i++) { - if (this.ws.readyState === WebSocket.OPEN) { + if (this.ws.readyState === status) { break; } await sleep(100); } - if (this.ws.readyState !== WebSocket.OPEN) { - throw new Error('Cannot connect to websocket!'); + if (this.ws.readyState !== status) { + throw new Error(errMessage); } - this.subscribeNewBlock(); } - send(method, params) { - if (this.closed) { - throw new Error('Trying to send with a closed explorer'); - } + async subscribe(method, params, callback) { + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); const id = this.ID.toString(); + this.subscriptions[id] = callback; const req = { id, method, params, }; - this.ID++; this.ws.send(JSON.stringify(req)); + this.ID++; return id; } + async subscribeNewBlock() { + await this.subscribe('subscribeNewBlock', {}, function (result) { + if (result['height'] !== undefined) { + getEventEmitter().emit('new-block', result['height']); + } + }); + } + + async send(method, params) { + // It might happen that connection just got closed, and we are still connecting to the new one. + // So make sure that before sending anything we have an open connection + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); + const id = this.ID.toString(); + const req = { + id, + method, + params, + }; + this.ID++; + this.ws.send(JSON.stringify(req)); + return [id, this.ws]; + } async sendAndWaitForAnswer(method, params) { let attempt = 1; const maxAttempts = 10; @@ -199,11 +252,8 @@ export class ExplorerNetwork extends Network { const frequency = 100; while (attempt <= maxAttempts) { let receivedInvalidAnswer = false; - const id = this.send(method, params); + const [id, ws_that_sent] = await this.send(method, params); for (let i = 0; i < Math.floor(maxAwaitTime / frequency); i++) { - if (this.closed) { - break; - } const res = this.cachedResults[id]; if (res !== undefined) { delete this.cachedResults[id]; @@ -215,6 +265,10 @@ export class ExplorerNetwork extends Network { return res; } await sleep(frequency); + // If connection got closed while sending do not wait until timeout + if (ws_that_sent !== this.ws) { + break; + } } debugWarn( DebugTopics.NET, @@ -228,28 +282,7 @@ export class ExplorerNetwork extends Network { ); attempt += 1; } - if (!this.closed) { - throw new Error('Failed to communicate with the explorer'); - } - } - subscribe(method, params, callback) { - const id = this.ID.toString(); - this.subscriptions[id] = callback; - const req = { - id, - method, - params, - }; - this.ws.send(JSON.stringify(req)); - this.ID++; - return id; - } - subscribeNewBlock() { - this.subscribe('subscribeNewBlock', {}, function (result) { - if (result['height'] !== undefined) { - getEventEmitter().emit('new-block', result['height']); - } - }); + throw new Error('Failed to communicate with the explorer'); } async getAccountInfo(descriptor, page, pageSize, from, details = 'txs') { const params = { @@ -457,17 +490,15 @@ export class ExplorerNetwork extends Network { } } -let _network = null; +let _network = new ExplorerNetwork(); /** * Sets the network in use by MPW. - * @param {ExplorerNetwork} network - network to use + * @param {String} wsUrl - websocket url */ -export async function setNetwork(network) { - debugLog(DebugTopics.NET, 'Connecting to new explorer', network.wsUrl); - _network?.close(); - _network = network; - await _network.init(); +export async function setNetwork(wsUrl) { + debugLog(DebugTopics.NET, 'Connecting to new explorer', wsUrl); + await _network.createWebSocketConnection(wsUrl); } /** @@ -475,5 +506,6 @@ export async function setNetwork(network) { * @returns {ExplorerNetwork?} Returns the network in use, may be null if MPW hasn't properly loaded yet. */ export function getNetwork() { - return _network; + // Return null if websocket hasn't been loaded yet + return _network.ws ? _network : null; } diff --git a/scripts/settings.js b/scripts/settings.js index 08c8ea9c9..a9e943a34 100644 --- a/scripts/settings.js +++ b/scripts/settings.js @@ -308,8 +308,7 @@ async function setExplorer(explorer, fSilent = false) { cExplorer = explorer; // Enable networking + notify if allowed - const network = new ExplorerNetwork(cExplorer.url); - await setNetwork(network); + await setNetwork(cExplorer.url); // Update the selector UI doms.domExplorerSelect.value = cExplorer.url; From 01b1aa5310d6d73655658e1d6cf2cddd1a9c938c Mon Sep 17 00:00:00 2001 From: Alessandro Rezzi Date: Thu, 16 May 2024 23:15:11 +0200 Subject: [PATCH 10/10] half the shield batch size --- scripts/chain_params.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/chain_params.js b/scripts/chain_params.js index 329c4c076..b26cb5aef 100644 --- a/scripts/chain_params.js +++ b/scripts/chain_params.js @@ -13,7 +13,7 @@ export const COIN = 10 ** 8; export const MAX_ACCOUNT_GAP = 20; /** The batch size of Shield block synchronisation */ -export const SHIELD_BATCH_SYNC_SIZE = 32; +export const SHIELD_BATCH_SYNC_SIZE = 16; /** Transaction Sapling Version */ export const SAPLING_TX_VERSION = 3;