diff --git a/scripts/__mocks__/global.js b/scripts/__mocks__/global.js index 67b053520..dcc8c3a6e 100644 --- a/scripts/__mocks__/global.js +++ b/scripts/__mocks__/global.js @@ -1 +1,14 @@ -export const blockCount = 1504903; +import { getEventEmitter } from '../event_bus.js'; +import { vi } from 'vitest'; + +export let blockCount = 1504903; + +export const start = vi.fn(() => { + subscribeToNetworkEvents(); +}); + +const subscribeToNetworkEvents = vi.fn(() => { + getEventEmitter().on('new-block', (block) => { + blockCount = block; + }); +}); diff --git a/scripts/__mocks__/network.js b/scripts/__mocks__/network.js index bd393b50a..bca99b184 100644 --- a/scripts/__mocks__/network.js +++ b/scripts/__mocks__/network.js @@ -1,5 +1,7 @@ import { vi } from 'vitest'; import { Transaction } from '../transaction.js'; +import { getEventEmitter } from '../event_bus.js'; +import { sleep } from '../utils.js'; export const getNetwork = vi.fn(() => { return globalNetwork; @@ -106,6 +108,10 @@ class TestNetwork { this.#nextBlock.reset(); this.#nextBlock.blockHeight = this.#blockHeight + 1; } + mintAndEmit() { + this.mintBlock(); + getEventEmitter().emit('new-block', this.#blockHeight); + } } /** @@ -157,6 +163,9 @@ class TestTransaction { let globalNetwork = new TestNetwork(); -export function resetNetwork() { +export async function resetNetwork() { globalNetwork = new TestNetwork(); + // Update the global variable blockCount + getEventEmitter().emit('new-block', globalNetwork.getBlockCount()); + await sleep(100); } diff --git a/scripts/chain_params.js b/scripts/chain_params.js index 329c4c076..b26cb5aef 100644 --- a/scripts/chain_params.js +++ b/scripts/chain_params.js @@ -13,7 +13,7 @@ export const COIN = 10 ** 8; export const MAX_ACCOUNT_GAP = 20; /** The batch size of Shield block synchronisation */ -export const SHIELD_BATCH_SYNC_SIZE = 32; +export const SHIELD_BATCH_SYNC_SIZE = 16; /** Transaction Sapling Version */ export const SAPLING_TX_VERSION = 3; diff --git a/scripts/global.js b/scripts/global.js index eba3ed722..e0375aefa 100644 --- a/scripts/global.js +++ b/scripts/global.js @@ -247,16 +247,13 @@ export async function start() { subscribeToNetworkEvents(); // Make sure we know the correct number of blocks - await refreshChainData(); + await updateBlockCount(); // Load the price manager cOracle.load(); // If allowed by settings: submit a simple 'hit' (app load) to Labs Analytics getNetwork().submitAnalytics('hit'); setInterval(() => { - // Refresh blockchain data - refreshChainData(); - // Fetch the PIVX prices refreshPriceDisplay(); }, 15000); @@ -274,6 +271,14 @@ export async function start() { doms.domDashboard.click(); } +/** + * Updates the blockcount + * Must be called only on start and when toggling network + * @returns {Promise} + */ +export async function updateBlockCount() { + blockCount = await getNetwork().getBlockCount(); +} async function refreshPriceDisplay() { await cOracle.getPrice(strCurrency); getEventEmitter().emit('balance-update'); @@ -287,7 +292,7 @@ function subscribeToNetworkEvents() { getEventEmitter().on('new-block', (block) => { debugLog(DebugTopics.GLOBAL, `New block detected! ${block}`); - + blockCount = block; // If it's open: update the Governance Dashboard if (doms.domGovTab.classList.contains('active')) { updateGovernanceTab(); @@ -1654,22 +1659,6 @@ export async function createProposal() { } } -export async function refreshChainData() { - const cNet = getNetwork(); - // If in offline mode: don't sync ANY data or connect to the internet - if (!cNet.enabled) - return console.warn( - 'Offline mode active: For your security, the wallet will avoid ALL internet requests.' - ); - - // Fetch block count - const newBlockCount = await cNet.getBlockCount(); - if (newBlockCount !== blockCount) { - blockCount = newBlockCount; - getEventEmitter().emit('new-block', blockCount); - } -} - // A safety mechanism enabled if the user attempts to leave without encrypting/saving their keys export const beforeUnloadListener = (evt) => { evt.preventDefault(); diff --git a/scripts/network.js b/scripts/network.js index f50c979d1..97c89c020 100644 --- a/scripts/network.js +++ b/scripts/network.js @@ -1,10 +1,11 @@ -import { cChainParams } from './chain_params.js'; import { createAlert } from './misc.js'; import { + debugError, debugLog, debugTimerEnd, debugTimerStart, DebugTopics, + debugWarn, } from './debug.js'; import { sleep } from './utils.js'; import { getEventEmitter } from './event_bus.js'; @@ -12,8 +13,7 @@ import { STATS, cStatKeys, cAnalyticsLevel, - setExplorer, - fAutoSwitch, + setNextExplorer, } from './settings.js'; import { cNode } from './settings.js'; import { ALERTS, tr, translation } from './i18n.js'; @@ -107,20 +107,192 @@ export class Network { } } -/** - * - */ export class ExplorerNetwork extends Network { /** - * @param {string} strUrl - Url pointing to the blockbook explorer + * @param {string} wsUrl - Url pointing to the blockbook explorer */ - constructor(strUrl, wallet) { - super(wallet); - /** - * @type{string} - * @public - */ - this.strUrl = strUrl; + constructor() { + super(); + this.cachedResults = []; + this.subscriptions = []; + this.ID = 0; + } + async createWebSocketConnection(wsUrl) { + // Make sure the old connection is closed before opening a new one + if (this.ws) { + await this.reset(); + } + // ensure backward compatibility + if (wsUrl.startsWith('http')) { + wsUrl = wsUrl.replace('http', 'ws'); + } + if (!wsUrl.endsWith('/websocket')) { + wsUrl += '/websocket'; + } + this.wsUrl = wsUrl; + this.openWebSocketConnection(wsUrl); + await this.initWebSocketConnection(); + } + openWebSocketConnection(wsUrl) { + this.ws = new WebSocket(wsUrl); + this.ws.onopen = function (e) { + debugLog(DebugTopics.NET, 'websocket connected', e); + }; + this.ws.onclose = function (e) { + debugLog(DebugTopics.NET, 'websocket disconnected', e); + if (!e.wasClean) { + debugError( + DebugTopics.NET, + 'websocket unexpected close, trying to reconnect' + ); + // Connection closed somehow, try to reconnect + setNextExplorer(); + } + }; + this.ws.onmessage = function (e) { + // Return early if the websocket is not open + // In this way we avoid the case in which a closing websocket connection put values in the cache + if (this.readyState !== WebSocket.OPEN) { + return; + } + const resp = JSON.parse(e.data); + // Is this a subscription? + const f = _network.subscriptions[resp.id]; + if (f !== undefined) { + f(resp.data); + } + // If it isn't cache the result + _network.cachedResults[resp.id] = resp.data; + }; + } + async initWebSocketConnection() { + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); + await this.subscribeNewBlock(); + } + get strUrl() { + return this.wsUrl.replace('ws', 'http').replace('/websocket', ''); + } + + async reset() { + if (this.ws.readyState == WebSocket.OPEN) { + this.ws.close(); + } + // Make sure websocket got closed before emptying arrays + await this.awaitWebSocketStatus( + WebSocket.CLOSED, + 'Websocket is not getting closed' + ); + // At this point messages of old websocket will not be received anymore + // and, it is safe to reset cachedResults and subscriptions + this.cachedResults = []; + this.subscriptions = []; + } + + async awaitWebSocketStatus(status, errMessage) { + for (let i = 0; i < 100; i++) { + if (this.ws.readyState === status) { + break; + } + await sleep(100); + } + if (this.ws.readyState !== status) { + throw new Error(errMessage); + } + } + + async subscribe(method, params, callback) { + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); + const id = this.ID.toString(); + this.subscriptions[id] = callback; + const req = { + id, + method, + params, + }; + this.ws.send(JSON.stringify(req)); + this.ID++; + return id; + } + async subscribeNewBlock() { + await this.subscribe('subscribeNewBlock', {}, function (result) { + if (result['height'] !== undefined) { + getEventEmitter().emit('new-block', result['height']); + } + }); + } + + async send(method, params) { + // It might happen that connection just got closed, and we are still connecting to the new one. + // So make sure that before sending anything we have an open connection + await this.awaitWebSocketStatus( + WebSocket.OPEN, + 'Cannot connect to websocket!' + ); + const id = this.ID.toString(); + const req = { + id, + method, + params, + }; + this.ID++; + this.ws.send(JSON.stringify(req)); + return [id, this.ws]; + } + async sendAndWaitForAnswer(method, params) { + let attempt = 1; + const maxAttempts = 10; + // milliseconds + const maxAwaitTime = 600 * 1000; + const frequency = 100; + while (attempt <= maxAttempts) { + let receivedInvalidAnswer = false; + const [id, ws_that_sent] = await this.send(method, params); + for (let i = 0; i < Math.floor(maxAwaitTime / frequency); i++) { + const res = this.cachedResults[id]; + if (res !== undefined) { + delete this.cachedResults[id]; + if (res.error) { + await sleep(1000); + receivedInvalidAnswer = true; + break; + } + return res; + } + await sleep(frequency); + // If connection got closed while sending do not wait until timeout + if (ws_that_sent !== this.ws) { + break; + } + } + debugWarn( + DebugTopics.NET, + 'Failed send attempt for ' + method, + 'attempt ' + + attempt + + '/' + + maxAttempts + + ' received an invalid answer: ' + + receivedInvalidAnswer + ); + attempt += 1; + } + throw new Error('Failed to communicate with the explorer'); + } + async getAccountInfo(descriptor, page, pageSize, from, details = 'txs') { + const params = { + descriptor, + details, + page, + pageSize, + from, + }; + return await this.sendAndWaitForAnswer('getAccountInfo', params); } error() { @@ -136,76 +308,17 @@ export class ExplorerNetwork extends Network { * @param {boolean} skipCoinstake - if true coinstake tx will be skipped * @returns {Promise} the block fetched from explorer */ - async getBlock(blockHeight, skipCoinstake = false) { - try { - const block = await this.safeFetchFromExplorer( - `/api/v2/block/${blockHeight}` - ); - const newTxs = []; - // This is bad. We're making so many requests - // This is a quick fix to try to be compliant with the blockbook - // API, and not the PIVX extension. - // In the Blockbook API /block doesn't have any chain specific information - // Like hex, shield info or what not. - // We could change /getshieldblocks to /getshieldtxs? - // In addition, always skip the coinbase transaction and in case the coinstake one - // TODO: once v6.0 and shield stake is activated we might need to change this optimization - for (const tx of block.txs.slice(skipCoinstake ? 2 : 1)) { - const r = await fetch( - `${this.strUrl}/api/v2/tx-specific/${tx.txid}` - ); - if (!r.ok) throw new Error('failed'); - const newTx = await r.json(); - newTxs.push(newTx); - } - block.txs = newTxs; - return block; - } catch (e) { - this.error(); - throw e; - } - } - - async getBlockCount() { + async getBlock(blockNumber) { try { - const { backend } = await ( - await retryWrapper(fetchBlockbook, `/api/v2/api`) - ).json(); - - return backend.blocks; + return await this.sendAndWaitForAnswer('getBlock', { + id: blockNumber.toString(), + }); } catch (e) { this.error(); throw e; } } - /** - * Sometimes blockbook might return internal error, in this case this function will sleep for some times and retry - * @param {string} strCommand - The specific Blockbook api to call - * @param {number} sleepTime - How many milliseconds sleep between two calls. Default value is 20000ms - * @returns {Promise} Explorer result in json - */ - async safeFetchFromExplorer(strCommand, sleepTime = 20000) { - let trials = 0; - const maxTrials = 6; - while (trials < maxTrials) { - trials += 1; - const res = await retryWrapper(fetchBlockbook, strCommand); - if (!res.ok) { - debugLog( - DebugTopics.NET, - 'Blockbook internal error! sleeping for ' + - sleepTime + - ' seconds' - ); - await sleep(sleepTime); - continue; - } - return await res.json(); - } - throw new Error('Cannot safe fetch from explorer!'); - } - /** * //TODO: do not take the wallet as parameter but instead something weaker like a public key or address? * Must be called only for initial wallet sync @@ -220,14 +333,11 @@ export class ExplorerNetwork extends Network { ...wallet.getTransactions().map((tx) => tx.blockHeight) ); debugTimerStart(DebugTopics.NET, 'getLatestTxsTimer'); - // Form the API call using our wallet information - const strKey = wallet.getKeyToExport(); - const strRoot = `/api/v2/${ - wallet.isHD() ? 'xpub/' : 'address/' - }${strKey}`; - const strCoreParams = `?details=txs&from=${nStartHeight}`; - const probePage = await this.safeFetchFromExplorer( - `${strRoot + strCoreParams}&pageSize=1` + const probePage = await this.getAccountInfo( + wallet.getKeyToExport(), + 1, + 1, + nStartHeight ); const txNumber = probePage.txs - wallet.getTransactions().length; // Compute the total pages and iterate through them until we've synced everything @@ -243,8 +353,11 @@ export class ExplorerNetwork extends Network { ); // Fetch this page of transactions - const iPage = await this.safeFetchFromExplorer( - `${strRoot + strCoreParams}&page=${i}` + const iPage = await this.getAccountInfo( + wallet.getKeyToExport(), + i, + 1000, + nStartHeight ); // Update the internal mempool if there's new transactions @@ -282,17 +395,9 @@ export class ExplorerNetwork extends Network { * @returns {Promise>} Resolves when it has finished fetching UTXOs */ async getUTXOs(strAddress) { - try { - let publicKey = strAddress; - // Fetch UTXOs for the key - const arrUTXOs = await ( - await retryWrapper(fetchBlockbook, `/api/v2/utxo/${publicKey}`) - ).json(); - return arrUTXOs; - } catch (e) { - console.error(e); - this.error(); - } + return await this.sendAndWaitForAnswer('getAccountUtxo', { + descriptor: strAddress, + }); } /** @@ -301,19 +406,14 @@ export class ExplorerNetwork extends Network { * @returns {Promise} - A JSON class of aggregated XPUB info */ async getXPubInfo(strXPUB) { - return await ( - await retryWrapper(fetchBlockbook, `/api/v2/xpub/${strXPUB}`) - ).json(); + return await this.getAccountInfo(strXPUB, 1, 1, 0, 'tokens'); } async sendTransaction(hex) { try { - const data = await ( - await retryWrapper(fetchBlockbook, '/api/v2/sendtx/', { - method: 'post', - body: hex, - }) - ).json(); + const data = await this.sendAndWaitForAnswer('sendTransaction', { + hex, + }); // Throw and catch if the data is not a TXID if (!data.result || data.result.length !== 64) throw data; @@ -328,8 +428,26 @@ export class ExplorerNetwork extends Network { } async getTxInfo(txHash) { - const req = await retryWrapper(fetchBlockbook, `/api/v2/tx/${txHash}`); - return await req.json(); + return await this.sendAndWaitForAnswer('getTransaction', { + txid: txHash, + }); + } + + /** + * Get the blockchain info. + * Internal function used to get the blockchain info. + */ + async #getChainInfo() { + return await this.sendAndWaitForAnswer('getInfo', {}); + } + + /** + * Returns the best known block height. + * Must be called only on start and when toggling mainnet/testnet + * @returns {Promise} + */ + async getBlockCount() { + return (await this.#getChainInfo())['bestHeight']; } /** @@ -372,14 +490,15 @@ export class ExplorerNetwork extends Network { } } -let _network = null; +let _network = new ExplorerNetwork(); /** * Sets the network in use by MPW. - * @param {ExplorerNetwork} network - network to use + * @param {String} wsUrl - websocket url */ -export function setNetwork(network) { - _network = network; +export async function setNetwork(wsUrl) { + debugLog(DebugTopics.NET, 'Connecting to new explorer', wsUrl); + await _network.createWebSocketConnection(wsUrl); } /** @@ -387,67 +506,6 @@ export function setNetwork(network) { * @returns {ExplorerNetwork?} Returns the network in use, may be null if MPW hasn't properly loaded yet. */ export function getNetwork() { - return _network; -} - -/** - * A Fetch wrapper which uses the current Blockbook Network's base URL - * @param {string} api - The specific Blockbook api to call - * @param {RequestInit} options - The Fetch options - * @returns {Promise} - The unresolved Fetch promise - */ -export function fetchBlockbook(api, options) { - return fetch(_network.strUrl + api, options); -} - -/** - * A wrapper for Blockbook calls which can, in the event of an unresponsive explorer, - * seamlessly attempt the same call on multiple other explorers until success. - * @param {Function} func - The function to re-attempt with - * @param {...any} args - The arguments to pass to the function - */ -async function retryWrapper(func, ...args) { - // Track internal errors from the wrapper - let err; - - // If allowed by the user, Max Tries is ALL MPW-supported explorers, otherwise, restrict to only the current one. - let nMaxTries = cChainParams.current.Explorers.length; - let retries = 0; - - // The explorer index we started at - let nIndex = cChainParams.current.Explorers.findIndex( - (a) => a.url === getNetwork().strUrl - ); - - // Run the call until successful, or all attempts exhausted - while (retries < nMaxTries) { - try { - // Call the passed function with the arguments - const res = await func(...args); - - // If the endpoint is non-OK, assume it's an error - if (!res.ok) throw res; - - // Return the result if successful - return res; - } catch (error) { - err = error; - - // If allowed, switch explorers - if (!fAutoSwitch) throw err; - nIndex = (nIndex + 1) % cChainParams.current.Explorers.length; - const cNewExplorer = cChainParams.current.Explorers[nIndex]; - - // Set the explorer at Network-class level, then as a hacky workaround for the current callback; we - // ... adjust the internal URL to the new explorer. - getNetwork().strUrl = cNewExplorer.url; - setExplorer(cNewExplorer, true); - - // Bump the attempts, and re-try next loop - retries++; - } - } - - // Throw an error so the calling code knows the operation failed - throw err; + // Return null if websocket hasn't been loaded yet + return _network.ws ? _network : null; } diff --git a/scripts/settings.js b/scripts/settings.js index 026d78914..a9e943a34 100644 --- a/scripts/settings.js +++ b/scripts/settings.js @@ -3,7 +3,7 @@ import { updateLogOutButton, updateGovernanceTab, dashboard, - refreshChainData, + updateBlockCount, } from './global.js'; import { wallet, hasEncryptedWallet } from './wallet.js'; import { cChainParams } from './chain_params.js'; @@ -302,18 +302,13 @@ function subscribeToNetworkEvents() { } // --- Settings Functions -export async function setExplorer(explorer, fSilent = false) { +async function setExplorer(explorer, fSilent = false) { const database = await Database.getInstance(); database.setSettings({ explorer: explorer.url }); cExplorer = explorer; // Enable networking + notify if allowed - if (getNetwork()) { - getNetwork().strUrl = cExplorer.url; - } else { - const network = new ExplorerNetwork(cExplorer.url, wallet); - setNetwork(network); - } + await setNetwork(cExplorer.url); // Update the selector UI doms.domExplorerSelect.value = cExplorer.url; @@ -326,6 +321,18 @@ export async function setExplorer(explorer, fSilent = false) { ); } +export async function setNextExplorer() { + // The explorer index we started at + let nIndex = cChainParams.current.Explorers.findIndex( + (a) => a.url === getNetwork().strUrl + ); + if (fAutoSwitch) { + nIndex = (nIndex + 1) % cChainParams.current.Explorers.length; + } + const cNewExplorer = cChainParams.current.Explorers[nIndex]; + await setExplorer(cNewExplorer, true); +} + async function setNode(node, fSilent = false) { cNode = node; const database = await Database.getInstance(); @@ -562,7 +569,7 @@ export async function toggleTestnet() { doms.domTestnetToggler.checked = cChainParams.current.isTestnet; await start(); // Make sure we have the correct number of blocks before loading any wallet - await refreshChainData(); + await updateBlockCount(); getEventEmitter().emit('toggle-network'); await updateGovernanceTab(); } diff --git a/scripts/wallet.js b/scripts/wallet.js index 81eb1bdc5..68d2cd988 100644 --- a/scripts/wallet.js +++ b/scripts/wallet.js @@ -737,7 +737,7 @@ export class Wallet { await startBatch( async (i) => { let block; - block = await cNet.getBlock(blockHeights[i], true); + block = await cNet.getBlock(blockHeights[i]); blocks[i] = block; // We need to process blocks monotically // When we get a block, start from the first unhandled @@ -750,17 +750,16 @@ export class Wallet { // Delete so we don't have to hold all blocks in memory // until we finish syncing delete blocks[j]; + getEventEmitter().emit( + 'shield-sync-status-update', + tr(translation.syncShieldProgress, [ + { current: handled }, + { total: blockHeights.length }, + ]), + false + ); syncing = false; } - - getEventEmitter().emit( - 'shield-sync-status-update', - tr(translation.syncShieldProgress, [ - { current: handled - 1 }, - { total: blockHeights.length }, - ]), - false - ); }, blockHeights.length, batchSize @@ -841,7 +840,7 @@ export class Wallet { for (const tx of block.txs) { const parsed = Transaction.fromHex(tx.hex); parsed.blockHeight = blockHeight; - parsed.blockTime = tx.blocktime; + parsed.blockTime = tx.blockTime; // Avoid wasting memory on txs that do not regard our wallet if (this.ownTransaction(parsed)) { await this.addTransaction(parsed); diff --git a/tests/integration/wallet/sync.spec.js b/tests/integration/wallet/sync.spec.js index 9a5f4f7d0..3a283d1cb 100644 --- a/tests/integration/wallet/sync.spec.js +++ b/tests/integration/wallet/sync.spec.js @@ -9,10 +9,11 @@ import { getNetwork, resetNetwork, } from '../../../scripts/__mocks__/network.js'; -import { refreshChainData } from '../../../scripts/global.js'; import { sleep } from '../../../scripts/utils.js'; +import { start } from '../../../scripts/__mocks__/global.js'; vi.mock('../../../scripts/network.js'); +vi.mock('../../../scripts/global.js'); /** * @param{import('scripts/wallet').Wallet} wallet - wallet that will generate the transaction @@ -28,8 +29,7 @@ async function crateAndSendTransaction(wallet, address, value) { } async function mineAndSync() { - getNetwork().mintBlock(); - await refreshChainData(); + getNetwork().mintAndEmit(); // 500 milliseconds are enough time to make the wallets sync and handle the new blocks await sleep(500); } @@ -37,10 +37,11 @@ async function mineAndSync() { describe('Wallet sync tests', () => { let walletHD; let walletLegacy; + beforeAll(async () => { + await start(); + }); beforeEach(async () => { - resetNetwork(); - // Update the global variable blockCount - await refreshChainData(); + await resetNetwork(); walletHD = await setUpHDMainnetWallet(false); walletLegacy = await setUpLegacyMainnetWallet(); // Reset indexedDB before each test