From 214441b4a87c9fd06dcc5afd95065323228e82f5 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:40:26 -0700 Subject: [PATCH 1/7] add bor and heimdall services back to polygon --- .circleci/config.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 827fea7e5..860a67d3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -609,12 +609,26 @@ aliases: api-memory-limit: 1Gi api-memory-request: 500Mi stateful-service-replicas: 2 - service-name-1: indexer - service-image-1: shapeshiftdao/unchained-blockbook:559cfbc - service-cpu-limit-1: "4" - service-cpu-request-1: "2" - service-memory-limit-1: 24Gi - service-storage-size-1: 750Gi + service-name-1: daemon + service-image-1: 0xpolygon/bor:1.2.0 + service-cpu-limit-1: "8" + service-cpu-request-1: "4" + service-memory-limit-1: 48Gi + service-storage-size-1: 4000Gi + service-storage-iops-1: "6000" + service-storage-throughput-1: "300" + service-name-2: heimdall + service-image-2: 0xpolygon/heimdall:1.0.3 + service-cpu-limit-2: "2" + service-cpu-request-2: "1" + service-memory-limit-2: 1Gi + service-storage-size-2: 400Gi + service-name-3: indexer + service-image-3: shapeshiftdao/unchained-blockbook:559cfbc + service-cpu-limit-3: "4" + service-cpu-request-3: "2" + service-memory-limit-3: 24Gi + service-storage-size-3: 750Gi - &polygon-dev <<: *polygon From 32a8ff946826f6be3ebe32917b45b74069d04a0f Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:42:16 -0700 Subject: [PATCH 2/7] add logger to blockbook and update with debug logs for future use --- .../arbitrum-nova/api/src/controller.ts | 2 +- .../coinstacks/arbitrum/api/src/controller.ts | 2 +- .../avalanche/api/src/controller.ts | 2 +- node/coinstacks/bitcoin/api/src/controller.ts | 8 +- .../bitcoincash/api/src/controller.ts | 8 +- .../bnbsmartchain/api/src/controller.ts | 2 +- .../coinstacks/dogecoin/api/src/controller.ts | 8 +- .../coinstacks/ethereum/api/src/controller.ts | 2 +- node/coinstacks/gnosis/api/src/controller.ts | 2 +- .../coinstacks/litecoin/api/src/controller.ts | 8 +- .../coinstacks/optimism/api/src/controller.ts | 2 +- node/coinstacks/polygon/api/src/controller.ts | 2 +- node/packages/blockbook/src/controller.ts | 97 +++++++++---------- 13 files changed, 83 insertions(+), 62 deletions(-) diff --git a/node/coinstacks/arbitrum-nova/api/src/controller.ts b/node/coinstacks/arbitrum-nova/api/src/controller.ts index d1435203f..731894506 100644 --- a/node/coinstacks/arbitrum-nova/api/src/controller.ts +++ b/node/coinstacks/arbitrum-nova/api/src/controller.ts @@ -38,7 +38,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'arbitrum-nova' }) diff --git a/node/coinstacks/arbitrum/api/src/controller.ts b/node/coinstacks/arbitrum/api/src/controller.ts index 0dd26b548..cfcb885e3 100644 --- a/node/coinstacks/arbitrum/api/src/controller.ts +++ b/node/coinstacks/arbitrum/api/src/controller.ts @@ -38,7 +38,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'arbitrum' }) diff --git a/node/coinstacks/avalanche/api/src/controller.ts b/node/coinstacks/avalanche/api/src/controller.ts index 955794632..c76b91b14 100644 --- a/node/coinstacks/avalanche/api/src/controller.ts +++ b/node/coinstacks/avalanche/api/src/controller.ts @@ -38,7 +38,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'avalanche' }) diff --git a/node/coinstacks/bitcoin/api/src/controller.ts b/node/coinstacks/bitcoin/api/src/controller.ts index ec185eaf3..8f9603de4 100644 --- a/node/coinstacks/bitcoin/api/src/controller.ts +++ b/node/coinstacks/bitcoin/api/src/controller.ts @@ -2,6 +2,7 @@ import { bech32 } from 'bech32' import { Blockbook } from '@shapeshiftoss/blockbook' import { Service } from '../../../common/api/src/utxo/service' import { UTXO } from '../../../common/api/src/utxo/controller' +import { Logger } from '@shapeshiftoss/logger' const INDEXER_URL = process.env.INDEXER_URL const INDEXER_WS_URL = process.env.INDEXER_WS_URL @@ -11,7 +12,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set') if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set') if (!RPC_URL) throw new Error('RPC_URL env var not set') -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +export const logger = new Logger({ + namespace: ['unchained', 'coinstacks', 'bitcoin', 'api'], + level: process.env.LOG_LEVEL, +}) + +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const isXpub = (pubkey: string): boolean => { return pubkey.startsWith('xpub') || pubkey.startsWith('ypub') || pubkey.startsWith('zpub') diff --git a/node/coinstacks/bitcoincash/api/src/controller.ts b/node/coinstacks/bitcoincash/api/src/controller.ts index 8fbe8052c..ab6206def 100644 --- a/node/coinstacks/bitcoincash/api/src/controller.ts +++ b/node/coinstacks/bitcoincash/api/src/controller.ts @@ -2,6 +2,7 @@ import { bech32 } from 'bech32' import { Blockbook } from '@shapeshiftoss/blockbook' import { Service } from '../../../common/api/src/utxo/service' import { UTXO } from '../../../common/api/src/utxo/controller' +import { Logger } from '@shapeshiftoss/logger' const INDEXER_URL = process.env.INDEXER_URL const INDEXER_WS_URL = process.env.INDEXER_WS_URL @@ -11,7 +12,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set') if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set') if (!RPC_URL) throw new Error('RPC_URL env var not set') -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +export const logger = new Logger({ + namespace: ['unchained', 'coinstacks', 'bitcoincash', 'api'], + level: process.env.LOG_LEVEL, +}) + +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const isXpub = (pubkey: string): boolean => { return pubkey.startsWith('xpub') || pubkey.startsWith('ypub') || pubkey.startsWith('zpub') diff --git a/node/coinstacks/bnbsmartchain/api/src/controller.ts b/node/coinstacks/bnbsmartchain/api/src/controller.ts index 577063ee1..a71f23cae 100644 --- a/node/coinstacks/bnbsmartchain/api/src/controller.ts +++ b/node/coinstacks/bnbsmartchain/api/src/controller.ts @@ -38,7 +38,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'bnbsmartchain' }) diff --git a/node/coinstacks/dogecoin/api/src/controller.ts b/node/coinstacks/dogecoin/api/src/controller.ts index f8000bebc..6c8189d81 100644 --- a/node/coinstacks/dogecoin/api/src/controller.ts +++ b/node/coinstacks/dogecoin/api/src/controller.ts @@ -1,6 +1,7 @@ import { Blockbook } from '@shapeshiftoss/blockbook' import { Service } from '../../../common/api/src/utxo/service' import { UTXO } from '../../../common/api/src/utxo/controller' +import { Logger } from '@shapeshiftoss/logger' const INDEXER_URL = process.env.INDEXER_URL const INDEXER_WS_URL = process.env.INDEXER_WS_URL @@ -10,7 +11,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set') if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set') if (!RPC_URL) throw new Error('RPC_URL env var not set') -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +export const logger = new Logger({ + namespace: ['unchained', 'coinstacks', 'gnosis', 'api'], + level: process.env.LOG_LEVEL, +}) + +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const isXpub = (pubkey: string): boolean => { return pubkey.startsWith('dgub') diff --git a/node/coinstacks/ethereum/api/src/controller.ts b/node/coinstacks/ethereum/api/src/controller.ts index eafb9c236..3cf480f8e 100644 --- a/node/coinstacks/ethereum/api/src/controller.ts +++ b/node/coinstacks/ethereum/api/src/controller.ts @@ -40,7 +40,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'ethereum' }) diff --git a/node/coinstacks/gnosis/api/src/controller.ts b/node/coinstacks/gnosis/api/src/controller.ts index a7d449677..e424b6f2a 100644 --- a/node/coinstacks/gnosis/api/src/controller.ts +++ b/node/coinstacks/gnosis/api/src/controller.ts @@ -40,7 +40,7 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'ethereum' }) diff --git a/node/coinstacks/litecoin/api/src/controller.ts b/node/coinstacks/litecoin/api/src/controller.ts index 350bc9d66..6b548b911 100644 --- a/node/coinstacks/litecoin/api/src/controller.ts +++ b/node/coinstacks/litecoin/api/src/controller.ts @@ -2,6 +2,7 @@ import { bech32 } from 'bech32' import { Blockbook } from '@shapeshiftoss/blockbook' import { Service } from '../../../common/api/src/utxo/service' import { UTXO } from '../../../common/api/src/utxo/controller' +import { Logger } from '@shapeshiftoss/logger' const INDEXER_URL = process.env.INDEXER_URL const INDEXER_WS_URL = process.env.INDEXER_WS_URL @@ -11,7 +12,12 @@ if (!INDEXER_URL) throw new Error('INDEXER_URL env var not set') if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set') if (!RPC_URL) throw new Error('RPC_URL env var not set') -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +export const logger = new Logger({ + namespace: ['unchained', 'coinstacks', 'litecoin', 'api'], + level: process.env.LOG_LEVEL, +}) + +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const isXpub = (pubkey: string): boolean => { return pubkey.startsWith('Ltub') || pubkey.startsWith('Mtub') || pubkey.startsWith('zpub') diff --git a/node/coinstacks/optimism/api/src/controller.ts b/node/coinstacks/optimism/api/src/controller.ts index 9cc73b745..cd65a6ff8 100644 --- a/node/coinstacks/optimism/api/src/controller.ts +++ b/node/coinstacks/optimism/api/src/controller.ts @@ -37,7 +37,7 @@ export const logger = new Logger({ const CHAIN_ID: Record = { mainnet: 10 } -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) +const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'optimism' }) diff --git a/node/coinstacks/polygon/api/src/controller.ts b/node/coinstacks/polygon/api/src/controller.ts index 27c4f2cfe..fa84331ac 100644 --- a/node/coinstacks/polygon/api/src/controller.ts +++ b/node/coinstacks/polygon/api/src/controller.ts @@ -38,8 +38,8 @@ export const logger = new Logger({ level: process.env.LOG_LEVEL, }) -const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL }) const provider = new ethers.providers.JsonRpcProvider(RPC_URL) +export const blockbook = new Blockbook({ httpURL: INDEXER_URL, wsURL: INDEXER_WS_URL, logger }) export const gasOracle = new GasOracle({ logger, provider, coinstack: 'polygon' }) export const service = new Service({ diff --git a/node/packages/blockbook/src/controller.ts b/node/packages/blockbook/src/controller.ts index 3448bf1ee..b5f08260b 100644 --- a/node/packages/blockbook/src/controller.ts +++ b/node/packages/blockbook/src/controller.ts @@ -1,5 +1,5 @@ import axios, { AxiosError, AxiosInstance } from 'axios' -import axiosRetry from 'axios-retry' +import axiosRetry, { isNetworkOrIdempotentRequestError } from 'axios-retry' import { Controller, Example, Get, Path, Query, Route, Tags } from 'tsoa' import WebSocket from 'ws' import { @@ -17,22 +17,24 @@ import { Utxo, Xpub, } from './models' +import { Logger } from '@shapeshiftoss/logger' + +const defaultArgs: BlockbookArgs = { + httpURL: 'https://indexer.ethereum.shapeshift.com', + wsURL: 'wss://indexer.ethereum.shapeshift.com/websocket', + logger: new Logger({ namespace: ['unchained', 'blockbook'], level: process.env.LOG_LEVEL }), +} @Route('api/v2') @Tags('v2') export class Blockbook extends Controller { instance: AxiosInstance wsURL: string + logger: Logger - constructor( - args: BlockbookArgs = { - httpURL: 'https://indexer.ethereum.shapeshift.com', - wsURL: 'wss://indexer.ethereum.shapeshift.com/websocket', - }, - timeout?: number, - retries = 3 - ) { + constructor(args: BlockbookArgs = defaultArgs, timeout?: number, retries = 5) { super() + this.logger = args.logger.child({ namespace: ['blockbook'] }) this.wsURL = args.wsURL this.instance = axios.create({ timeout: timeout ?? 10000, @@ -42,7 +44,14 @@ export class Blockbook extends Controller { 'Content-Type': 'application/json', }, }) - axiosRetry(this.instance, { shouldResetTimeout: true, retries, retryDelay: axiosRetry.exponentialDelay }) + axiosRetry(this.instance, { + shouldResetTimeout: true, + retries, + retryDelay: axiosRetry.exponentialDelay, + retryCondition: (err) => + isNetworkOrIdempotentRequestError(err) || + (!!err.response && err.response.status >= 400 && err.response.status < 600), + }) } /** @@ -87,10 +96,7 @@ export class Blockbook extends Controller { const { data } = await this.instance.get(`api/v2`) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err } } @@ -114,10 +120,7 @@ export class Blockbook extends Controller { const { data } = await this.instance.get(`api/v2/block-index/${height}`) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err } } @@ -233,15 +236,15 @@ export class Blockbook extends Controller { }) @Get('tx/{txid}') async getTransaction(@Path() txid: string): Promise { + const start = Date.now() try { const { data } = await this.instance.get(`api/v2/tx/${txid}`) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.trace(`getTransaction: ${txid}: ${Date.now() - start}ms`) } } @@ -252,15 +255,15 @@ export class Blockbook extends Controller { */ @Get('tx-specific/{txid}') async getTransactionSpecific(@Path() txid: string): Promise { + const start = Date.now() try { const { data } = await this.instance.get(`api/v2/tx-specific/${txid}`) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.trace(`getTransactionSpecific: ${txid}: ${Date.now() - start}ms`) } } @@ -439,6 +442,7 @@ export class Blockbook extends Controller { @Query() details?: 'basic' | 'tokens' | 'tokenBalances' | 'txids' | 'txslight' | 'txs', @Query() contract?: string ): Promise
{ + const start = Date.now() try { const { data } = await this.instance.get
(`api/v2/address/${address}`, { params: { @@ -452,11 +456,10 @@ export class Blockbook extends Controller { }) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.debug(`getAddress: ${address} (page: ${page}, details: ${details}): ${Date.now() - start}ms`) } } @@ -590,6 +593,7 @@ export class Blockbook extends Controller { @Query() details?: 'basic' | 'tokens' | 'tokenBalances' | 'txids' | 'txs', @Query() tokens?: 'nonzero' | 'used' | 'derived' ): Promise { + const start = Date.now() try { const { data } = await this.instance.get(`api/v2/xpub/${xpub}`, { params: { @@ -603,11 +607,10 @@ export class Blockbook extends Controller { }) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.debug(`getXpub: ${xpub} (page: ${page}, details: ${details}): ${Date.now() - start}ms`) } } @@ -650,6 +653,7 @@ export class Blockbook extends Controller { ]) @Get('utxo/{account}') async getUtxo(@Path() account: string, @Query() confirmed?: boolean): Promise> { + const start = Date.now() try { const { data } = await this.instance.get>(`api/v2/utxo/${account}`, { params: { @@ -658,11 +662,10 @@ export class Blockbook extends Controller { }) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.debug(`getUtxo: ${account}: ${Date.now() - start}ms`) } } @@ -751,6 +754,7 @@ export class Blockbook extends Controller { }) @Get('block/{block}') async getBlock(@Path() block: string, @Query() page?: number): Promise { + const start = Date.now() try { const { data } = await this.instance.get(`api/v2/block/${block}`, { params: { @@ -759,11 +763,10 @@ export class Blockbook extends Controller { }) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err + } finally { + this.logger.trace(`getBlock: ${block} (page: ${page}): ${Date.now() - start}ms`) } } @@ -784,10 +787,7 @@ export class Blockbook extends Controller { const { data } = await this.instance.get(`api/v2/sendtx/${hex}`) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err } } @@ -848,10 +848,7 @@ export class Blockbook extends Controller { }) return data } catch (err) { - if (err instanceof AxiosError || err instanceof Error) { - throw new ApiError(err) - } - + if (err instanceof AxiosError || err instanceof Error) throw new ApiError(err) throw err } } From 625f0a2d0649fe3cabea7a492b8ea9c0a97078f9 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:43:17 -0700 Subject: [PATCH 3/7] use blockbook paginated block endpoint instead of per tx block fetch --- node/coinstacks/common/api/src/evm/service.ts | 38 +++++-------------- .../coinstacks/common/api/src/utxo/service.ts | 33 +++++----------- 2 files changed, 20 insertions(+), 51 deletions(-) diff --git a/node/coinstacks/common/api/src/evm/service.ts b/node/coinstacks/common/api/src/evm/service.ts index 2a557ec4f..192227d8a 100644 --- a/node/coinstacks/common/api/src/evm/service.ts +++ b/node/coinstacks/common/api/src/evm/service.ts @@ -19,7 +19,6 @@ import { } from './models' import { Cursor, - NodeBlock, DebugCallStack, ExplorerApiResponse, TraceCall, @@ -323,34 +322,17 @@ export class Service implements Omit, API { } } - async handleBlock(hash: string, retryCount = 0): Promise> { - const request: RPCRequest = { - jsonrpc: '2.0', - id: `eth_getBlockByHash-${hash}`, - method: 'eth_getBlockByHash', - params: [hash, false], - } - - const { data } = await axios.post(this.rpcUrl, request) - - if (data.error) throw new Error(`failed to get block: ${hash}: ${data.error.message}`) - - // retry if no results are returned, this typically means we queried a node that hasn't indexed the data yet - if (!data.result) { - if (retryCount >= 5) throw new Error(`failed to get block: ${hash}: ${JSON.stringify(data)}`) - retryCount++ - await exponentialDelay(retryCount) - return this.handleBlock(hash, retryCount) + async handleBlock(hash: string): Promise> { + try { + const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash) + for (let page = 1; page < totalPages; ++page) { + const data = await this.blockbook.getBlock(hash) + data.txs && txs.push(...data.txs) + } + return txs + } catch (err) { + throw handleError(err) } - - const block = data.result as NodeBlock - - // make best effort to fetch all transactions, but don't fail handling block if a single transaction fails - const txs = await Promise.allSettled(block.transactions.map((hash) => this.blockbook.getTransaction(hash))) - - return txs - .filter((tx): tx is PromiseFulfilledResult => tx.status === 'fulfilled') - .map((tx) => tx.value) } handleTransaction(tx: BlockbookTx): Tx { diff --git a/node/coinstacks/common/api/src/utxo/service.ts b/node/coinstacks/common/api/src/utxo/service.ts index da6842480..7f0030da4 100644 --- a/node/coinstacks/common/api/src/utxo/service.ts +++ b/node/coinstacks/common/api/src/utxo/service.ts @@ -1,9 +1,8 @@ import axios from 'axios' import axiosRetry from 'axios-retry' import { ApiError as BlockbookApiError, Blockbook, Tx as BlockbookTx } from '@shapeshiftoss/blockbook' -import { AddressFormatter, ApiError, BadRequestError, BaseAPI, Cursor, RPCRequest, RPCResponse, SendTxBody } from '../' +import { AddressFormatter, ApiError, BadRequestError, BaseAPI, Cursor, SendTxBody } from '../' import { Account, Address, API, NetworkFee, NetworkFees, RawTx, Tx, TxHistory, Utxo } from './models' -import { NodeBlock } from './types' import { validatePageSize } from '../utils' axiosRetry(axios, { retries: 5, retryDelay: axiosRetry.exponentialDelay }) @@ -31,12 +30,10 @@ export class Service implements Omit, API { readonly isXpub: (pubkey: string) => boolean private readonly blockbook: Blockbook - private readonly rpcUrl: string private formatAddress: AddressFormatter = (address: string) => address.toLowerCase() constructor(args: ServiceArgs) { this.blockbook = args.blockbook - this.rpcUrl = args.rpcUrl this.isXpub = args.isXpub if (args.addressFormatter) this.formatAddress = args.addressFormatter @@ -182,26 +179,16 @@ export class Service implements Omit, API { } async handleBlock(hash: string): Promise> { - const request: RPCRequest = { - jsonrpc: '2.0', - id: `getblock-${hash}`, - method: 'getblock', - params: [hash], + try { + const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash) + for (let page = 1; page < totalPages; ++page) { + const data = await this.blockbook.getBlock(hash) + data.txs && txs.push(...data.txs) + } + return txs + } catch (err) { + throw handleError(err) } - - const { data } = await axios.post(this.rpcUrl, request) - - if (data.error) throw new Error(`failed to get block: ${hash}: ${data.error.message}`) - if (!data.result) throw new Error(`failed to get block: ${hash}: ${JSON.stringify(data)}`) - - const block = data.result as NodeBlock - - // make best effort to fetch all transactions, but don't fail handling block if a single transaction fails - const txs = await Promise.allSettled(block.tx.map((hash) => this.blockbook.getTransaction(hash))) - - return txs - .filter((tx): tx is PromiseFulfilledResult => tx.status === 'fulfilled') - .map((tx) => tx.value) } handleTransaction(tx: BlockbookTx): Tx { From 05fd066127b106ce17fc93d8939480d5524e3a63 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:44:02 -0700 Subject: [PATCH 4/7] remove useless and potentially blocking awaits --- node/packages/blockbook/src/websocket.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node/packages/blockbook/src/websocket.ts b/node/packages/blockbook/src/websocket.ts index 3411f0bd5..84441a9e4 100644 --- a/node/packages/blockbook/src/websocket.ts +++ b/node/packages/blockbook/src/websocket.ts @@ -119,9 +119,9 @@ export class WebsocketClient { if ('hash' in res.data) { const newBlock = res.data if (Array.isArray(this.handleBlock)) { - await Promise.all(this.handleBlock.map(async (handleBlock) => handleBlock(newBlock))) + this.handleBlock.map(async (handleBlock) => handleBlock(newBlock)) } else { - await this.handleBlock(newBlock) + this.handleBlock(newBlock) } } return @@ -129,9 +129,9 @@ export class WebsocketClient { if ('txid' in res.data) { const newTx = res.data if (Array.isArray(this.handleTransaction)) { - await Promise.all(this.handleTransaction.map(async (handleTransaction) => handleTransaction(newTx))) + this.handleTransaction.map(async (handleTransaction) => handleTransaction(newTx)) } else { - await this.handleTransaction(newTx) + this.handleTransaction(newTx) } } return From 529d47ebffb4a35e2ff3b9c34adba91654ac9ac4 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:46:31 -0700 Subject: [PATCH 5/7] pass logger to websocket connection handler and small cleanup --- node/coinstacks/arbitrum-nova/api/src/app.ts | 2 +- node/coinstacks/arbitrum/api/src/app.ts | 2 +- node/coinstacks/avalanche/api/src/app.ts | 2 +- node/coinstacks/bitcoin/api/src/app.ts | 2 +- node/coinstacks/bitcoincash/api/src/app.ts | 2 +- node/coinstacks/bnbsmartchain/api/src/app.ts | 2 +- node/coinstacks/common/api/src/websocket.ts | 35 ++++++++++---------- node/coinstacks/dogecoin/api/src/app.ts | 2 +- node/coinstacks/ethereum/api/src/app.ts | 2 +- node/coinstacks/gnosis/api/src/app.ts | 2 +- node/coinstacks/litecoin/api/src/app.ts | 2 +- node/coinstacks/optimism/api/src/app.ts | 2 +- node/coinstacks/polygon/api/src/app.ts | 2 +- 13 files changed, 30 insertions(+), 29 deletions(-) diff --git a/node/coinstacks/arbitrum-nova/api/src/app.ts b/node/coinstacks/arbitrum-nova/api/src/app.ts index 99e919dc1..885cfeb42 100644 --- a/node/coinstacks/arbitrum-nova/api/src/app.ts +++ b/node/coinstacks/arbitrum-nova/api/src/app.ts @@ -80,7 +80,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/arbitrum/api/src/app.ts b/node/coinstacks/arbitrum/api/src/app.ts index f0c1e8a12..502e6c8c8 100644 --- a/node/coinstacks/arbitrum/api/src/app.ts +++ b/node/coinstacks/arbitrum/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/avalanche/api/src/app.ts b/node/coinstacks/avalanche/api/src/app.ts index e348d5870..f0500455f 100644 --- a/node/coinstacks/avalanche/api/src/app.ts +++ b/node/coinstacks/avalanche/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/bitcoin/api/src/app.ts b/node/coinstacks/bitcoin/api/src/app.ts index 92886c916..781302254 100644 --- a/node/coinstacks/bitcoin/api/src/app.ts +++ b/node/coinstacks/bitcoin/api/src/app.ts @@ -76,7 +76,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/bitcoincash/api/src/app.ts b/node/coinstacks/bitcoincash/api/src/app.ts index d19f4580e..139a29542 100644 --- a/node/coinstacks/bitcoincash/api/src/app.ts +++ b/node/coinstacks/bitcoincash/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/bnbsmartchain/api/src/app.ts b/node/coinstacks/bnbsmartchain/api/src/app.ts index 36ed48ee0..bdbc70b96 100644 --- a/node/coinstacks/bnbsmartchain/api/src/app.ts +++ b/node/coinstacks/bnbsmartchain/api/src/app.ts @@ -80,7 +80,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/common/api/src/websocket.ts b/node/coinstacks/common/api/src/websocket.ts index 4ece79906..8867fe0ec 100644 --- a/node/coinstacks/common/api/src/websocket.ts +++ b/node/coinstacks/common/api/src/websocket.ts @@ -42,18 +42,18 @@ export class ConnectionHandler { private readonly websocket: WebSocket private readonly registry: Registry private readonly prometheus: Prometheus + private readonly logger: Logger private readonly routes: Record - private readonly pingInterval = 10000 + private readonly pingIntervalMs = 10000 private pingTimeout?: NodeJS.Timeout private subscriptionIds = new Map() - private logger = new Logger({ namespace: ['unchained', 'coinstacks', 'common', 'api'], level: process.env.LOG_LEVEL }) - - private constructor(websocket: WebSocket, registry: Registry, prometheus: Prometheus) { + private constructor(websocket: WebSocket, registry: Registry, prometheus: Prometheus, logger: Logger) { this.clientId = v4() this.registry = registry this.prometheus = prometheus + this.logger = logger.child({ namespace: ['websocket'] }) this.routes = { txs: { subscribe: (subscriptionId: string, data?: TxsTopicData) => this.handleSubscribeTxs(subscriptionId, data), @@ -61,30 +61,31 @@ export class ConnectionHandler { }, } - const interval = setInterval(() => { - this.websocket.ping() - }, this.pingInterval) + this.pingTimeout = undefined + this.prometheus.metrics.websocketCount.inc() + this.websocket = websocket + this.websocket.ping() - this.heartbeat() + const pingInterval = setInterval(() => { + this.websocket.ping() + }, this.pingIntervalMs) - this.websocket = websocket this.websocket.onerror = (error) => { this.logger.error({ clientId: this.clientId, error, fn: 'ws.onerror' }, 'websocket error') - this.close(interval) + this.close(pingInterval) } this.websocket.onclose = ({ code, reason }) => { this.prometheus.metrics.websocketCount.dec() this.logger.debug({ clientId: this.clientId, code, reason, fn: 'ws.close' }, 'websocket closed') - this.close(interval) + this.close(pingInterval) } this.websocket.on('pong', () => this.heartbeat()) this.websocket.on('ping', () => this.websocket.pong()) this.websocket.onmessage = (event) => this.onMessage(event) } - static start(websocket: WebSocket, registry: Registry, prometheus: Prometheus): void { - prometheus.metrics.websocketCount.inc() - new ConnectionHandler(websocket, registry, prometheus) + static start(websocket: WebSocket, registry: Registry, prometheus: Prometheus, logger: Logger): void { + new ConnectionHandler(websocket, registry, prometheus, logger) } private heartbeat(): void { @@ -93,9 +94,9 @@ export class ConnectionHandler { } this.pingTimeout = setTimeout(() => { - this.logger.debug({ fn: 'pingTimeout' }, 'heartbeat failed') + this.logger.debug({ clientId: this.clientId, fn: 'pingTimeout' }, 'heartbeat failed') this.websocket.terminate() - }, this.pingInterval + 1000) + }, this.pingIntervalMs + 1000) } private sendError(message: string, subscriptionId: string): void { @@ -130,7 +131,7 @@ export class ConnectionHandler { } } } catch (err) { - this.logger.error(err, { fn: 'onMessage', event }, 'Error processing message') + this.logger.error(err, { clientId: this.clientId, fn: 'onMessage', event }, 'Error processing message') } } diff --git a/node/coinstacks/dogecoin/api/src/app.ts b/node/coinstacks/dogecoin/api/src/app.ts index 16083a35a..a9ba26f94 100644 --- a/node/coinstacks/dogecoin/api/src/app.ts +++ b/node/coinstacks/dogecoin/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/ethereum/api/src/app.ts b/node/coinstacks/ethereum/api/src/app.ts index 53f5bd30e..3b34b4fce 100644 --- a/node/coinstacks/ethereum/api/src/app.ts +++ b/node/coinstacks/ethereum/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/gnosis/api/src/app.ts b/node/coinstacks/gnosis/api/src/app.ts index d68bd73ef..2236f228c 100644 --- a/node/coinstacks/gnosis/api/src/app.ts +++ b/node/coinstacks/gnosis/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/litecoin/api/src/app.ts b/node/coinstacks/litecoin/api/src/app.ts index 597e603c4..77abcf7b0 100644 --- a/node/coinstacks/litecoin/api/src/app.ts +++ b/node/coinstacks/litecoin/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/optimism/api/src/app.ts b/node/coinstacks/optimism/api/src/app.ts index 4042f8c7e..8cff34dc4 100644 --- a/node/coinstacks/optimism/api/src/app.ts +++ b/node/coinstacks/optimism/api/src/app.ts @@ -81,7 +81,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { diff --git a/node/coinstacks/polygon/api/src/app.ts b/node/coinstacks/polygon/api/src/app.ts index 6673dd1b2..3052bd7ad 100644 --- a/node/coinstacks/polygon/api/src/app.ts +++ b/node/coinstacks/polygon/api/src/app.ts @@ -78,7 +78,7 @@ const server = app.listen(PORT, () => logger.info('Server started')) const wsServer = new Server({ server }) wsServer.on('connection', (connection) => { - ConnectionHandler.start(connection, registry, prometheus) + ConnectionHandler.start(connection, registry, prometheus, logger) }) new WebsocketClient(INDEXER_WS_URL, { From 1a406c08621e4bb24762b6354f5b9088db872f2d Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:47:02 -0700 Subject: [PATCH 6/7] update type --- node/packages/blockbook/src/models.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/packages/blockbook/src/models.ts b/node/packages/blockbook/src/models.ts index e57988946..67368df05 100644 --- a/node/packages/blockbook/src/models.ts +++ b/node/packages/blockbook/src/models.ts @@ -1,3 +1,4 @@ +import { Logger } from '@shapeshiftoss/logger' import axios, { AxiosError } from 'axios' /** @@ -307,4 +308,5 @@ export interface FeeResponse { export interface BlockbookArgs { httpURL: string wsURL: string + logger: Logger } From 2cac624159f563c215a3e8625485c203577ec740 Mon Sep 17 00:00:00 2001 From: kaladinlight <35275952+kaladinlight@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:56:44 -0700 Subject: [PATCH 7/7] thanks team, this sickness make me stoopid... time for bed --- node/coinstacks/common/api/src/evm/service.ts | 2 +- node/coinstacks/common/api/src/utxo/service.ts | 2 +- node/coinstacks/dogecoin/api/src/controller.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node/coinstacks/common/api/src/evm/service.ts b/node/coinstacks/common/api/src/evm/service.ts index 192227d8a..b08d71477 100644 --- a/node/coinstacks/common/api/src/evm/service.ts +++ b/node/coinstacks/common/api/src/evm/service.ts @@ -326,7 +326,7 @@ export class Service implements Omit, API { try { const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash) for (let page = 1; page < totalPages; ++page) { - const data = await this.blockbook.getBlock(hash) + const data = await this.blockbook.getBlock(hash, page) data.txs && txs.push(...data.txs) } return txs diff --git a/node/coinstacks/common/api/src/utxo/service.ts b/node/coinstacks/common/api/src/utxo/service.ts index 7f0030da4..bd293bad3 100644 --- a/node/coinstacks/common/api/src/utxo/service.ts +++ b/node/coinstacks/common/api/src/utxo/service.ts @@ -182,7 +182,7 @@ export class Service implements Omit, API { try { const { txs = [], totalPages = 1 } = await this.blockbook.getBlock(hash) for (let page = 1; page < totalPages; ++page) { - const data = await this.blockbook.getBlock(hash) + const data = await this.blockbook.getBlock(hash, page) data.txs && txs.push(...data.txs) } return txs diff --git a/node/coinstacks/dogecoin/api/src/controller.ts b/node/coinstacks/dogecoin/api/src/controller.ts index 6c8189d81..25fe3c86e 100644 --- a/node/coinstacks/dogecoin/api/src/controller.ts +++ b/node/coinstacks/dogecoin/api/src/controller.ts @@ -12,7 +12,7 @@ if (!INDEXER_WS_URL) throw new Error('INDEXER_WS_URL env var not set') if (!RPC_URL) throw new Error('RPC_URL env var not set') export const logger = new Logger({ - namespace: ['unchained', 'coinstacks', 'gnosis', 'api'], + namespace: ['unchained', 'coinstacks', 'dogecoin', 'api'], level: process.env.LOG_LEVEL, })