From 24f246411a9cf5b1960fb3ed0550ceba3b9dea92 Mon Sep 17 00:00:00 2001 From: Lukasz Cwik Date: Wed, 29 Nov 2023 09:10:26 -0800 Subject: [PATCH] [IND-481] Remove market cache. More clean-up in preparation of migration to SQL based block processor. --- .../__tests__/loops/market-refresher.test.ts | 31 -------- indexer/packages/postgres/src/index.ts | 1 - .../postgres/src/loops/market-refresher.ts | 76 ------------------- .../__tests__/caches/block-cache.test.ts | 5 -- .../__tests__/handlers/asset-handler.test.ts | 2 - .../handlers/perpetual-market-handler.test.ts | 3 - .../ender/__tests__/lib/on-message.test.ts | 4 - .../validators/asset-validator.test.ts | 3 +- .../perpetual-market-validator.test.ts | 3 +- .../services/ender/src/caches/block-cache.ts | 2 - .../handlers/markets/market-create-handler.ts | 2 - .../handlers/markets/market-modify-handler.ts | 5 +- .../markets/market-price-update-handler.ts | 3 +- indexer/services/ender/src/index.ts | 3 +- .../services/ender/src/lib/cache-manager.ts | 2 - 15 files changed, 5 insertions(+), 140 deletions(-) delete mode 100644 indexer/packages/postgres/__tests__/loops/market-refresher.test.ts delete mode 100644 indexer/packages/postgres/src/loops/market-refresher.ts diff --git a/indexer/packages/postgres/__tests__/loops/market-refresher.test.ts b/indexer/packages/postgres/__tests__/loops/market-refresher.test.ts deleted file mode 100644 index b913a02b38..0000000000 --- a/indexer/packages/postgres/__tests__/loops/market-refresher.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { MarketCreateObject } from '../../src'; -import { clearData, migrate, teardown } from '../../src/helpers/db-helpers'; -import { getMarketFromId, updateMarkets } from '../../src/loops/market-refresher'; -import { defaultMarket, defaultMarket2 } from '../helpers/constants'; -import { seedData } from '../helpers/mock-generators'; - -describe('marketRefresher', () => { - beforeAll(async () => { - await migrate(); - await seedData(); - await updateMarkets(); - }); - - afterAll(async () => { - await clearData(); - await teardown(); - }); - - describe('getMarketFromId', () => { - it.each([ - [defaultMarket], - [defaultMarket2], - ])('successfully get an market from id', (market: MarketCreateObject) => { - expect(getMarketFromId(market.id)).toEqual(expect.objectContaining(market)); - }); - - it('returns undefined if market does not exist', () => { - expect(() => getMarketFromId(1000)).toThrowError('Unable to find market with id: 1000'); - }); - }); -}); diff --git a/indexer/packages/postgres/src/index.ts b/indexer/packages/postgres/src/index.ts index f2546f00ee..ed38fd30ae 100644 --- a/indexer/packages/postgres/src/index.ts +++ b/indexer/packages/postgres/src/index.ts @@ -36,7 +36,6 @@ export * as ComplianceTable from './stores/compliance-table'; export * as perpetualMarketRefresher from './loops/perpetual-market-refresher'; export * as assetRefresher from './loops/asset-refresher'; -export * as marketRefresher from './loops/market-refresher'; export * as liquidityTierRefresher from './loops/liquidity-tier-refresher'; export * as uuid from './helpers/uuid'; diff --git a/indexer/packages/postgres/src/loops/market-refresher.ts b/indexer/packages/postgres/src/loops/market-refresher.ts deleted file mode 100644 index b7a1858387..0000000000 --- a/indexer/packages/postgres/src/loops/market-refresher.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { - stats, delay, logger, NodeEnv, -} from '@dydxprotocol-indexer/base'; - -import config from '../config'; -import * as MarketTable from '../stores/market-table'; -import { MarketFromDatabase, MarketsMap, Options } from '../types'; - -let idToMarket: MarketsMap = {}; - -/** - * Refresh loop to cache the list of all markets from the database in-memory. - */ -export async function start(): Promise { - for (;;) { - await updateMarkets(); - await delay(config.MARKET_REFRESHER_INTERVAL_MS); - } -} - -/** - * Updates in-memory map of markets. - */ -export async function updateMarkets(options?: Options): Promise { - const startTime: number = Date.now(); - const markets: MarketFromDatabase[] = await MarketTable.findAll( - {}, - [], - options || { readReplica: true }, - ); - - const tmpIdToMarket: Record = {}; - markets.forEach( - (market: MarketFromDatabase) => { - tmpIdToMarket[market.id] = market; - }, - ); - - idToMarket = tmpIdToMarket; - stats.timing(`${config.SERVICE_NAME}.loops.update_markets`, Date.now() - startTime); -} - -/** - * Updates the markets map with the specified market. - */ -export function updateMarket(market: MarketFromDatabase): void { - idToMarket[market.id] = market; -} - -/** - * Gets the market for a given id. - */ -export function getMarketFromId(id: number): MarketFromDatabase { - const market: MarketFromDatabase | undefined = idToMarket[id]; - if (market === undefined) { - const message: string = `Unable to find market with id: ${id}`; - logger.error({ - at: 'market-refresher#getMarketFromId', - message, - }); - throw new Error(message); - } - return market; -} - -export function getMarketsMap(): MarketsMap { - return idToMarket; -} - -export function clear(): void { - if (config.NODE_ENV !== NodeEnv.TEST) { - throw new Error('clear cannot be used in non-test env'); - } - - idToMarket = {}; -} diff --git a/indexer/services/ender/__tests__/caches/block-cache.test.ts b/indexer/services/ender/__tests__/caches/block-cache.test.ts index 3157a0f807..da719f9e94 100644 --- a/indexer/services/ender/__tests__/caches/block-cache.test.ts +++ b/indexer/services/ender/__tests__/caches/block-cache.test.ts @@ -4,7 +4,6 @@ import { OraclePriceTable, assetRefresher, dbHelpers, - marketRefresher, perpetualMarketRefresher, testConstants, testMocks, @@ -35,7 +34,6 @@ describe('block-cache', () => { clearCandlesMap(); perpetualMarketRefresher.clear(); assetRefresher.clear(); - marketRefresher.clear(); }); afterAll(async () => { @@ -80,7 +78,6 @@ describe('block-cache', () => { expect(getCandlesMap()).not.toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({}); expect(assetRefresher.getAssetsMap()).not.toEqual({}); - expect(marketRefresher.getMarketsMap()).not.toEqual({}); } }); }); @@ -92,7 +89,6 @@ describe('block-cache', () => { expect(getCandlesMap()).toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).toEqual({}); expect(assetRefresher.getAssetsMap()).toEqual({}); - expect(marketRefresher.getMarketsMap()).toEqual({}); await initializeAllCaches(); @@ -101,7 +97,6 @@ describe('block-cache', () => { expect(getCandlesMap()).not.toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({}); expect(assetRefresher.getAssetsMap()).not.toEqual({}); - expect(marketRefresher.getMarketsMap()).not.toEqual({}); }); }); }); diff --git a/indexer/services/ender/__tests__/handlers/asset-handler.test.ts b/indexer/services/ender/__tests__/handlers/asset-handler.test.ts index 028579b909..75f210031d 100644 --- a/indexer/services/ender/__tests__/handlers/asset-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/asset-handler.test.ts @@ -12,7 +12,6 @@ import { MarketTable, Ordering, testConstants, - marketRefresher, BlockTable, TendermintEventTable, assetRefresher, @@ -112,7 +111,6 @@ describe('assetHandler', () => { it('creates new asset', async () => { await MarketTable.create(testConstants.defaultMarket); - await marketRefresher.updateMarkets(); const transactionIndex: number = 0; const assetEvent: AssetCreateEventV1 = defaultAssetCreateEvent; diff --git a/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts b/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts index 899ad0eb58..336ced9392 100644 --- a/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/perpetual-market-handler.test.ts @@ -12,7 +12,6 @@ import { MarketTable, Ordering, testConstants, - marketRefresher, BlockTable, TendermintEventTable, perpetualMarketRefresher, @@ -115,7 +114,6 @@ describe('perpetualMarketHandler', () => { it('fails when liquidity tier doesnt exist for perpetual market', async () => { await MarketTable.create(testConstants.defaultMarket); - await marketRefresher.updateMarkets(); const transactionIndex: number = 0; const kafkaMessage: KafkaMessage = createKafkaMessageFromPerpetualMarketEvent({ perpetualMarketEvent: defaultPerpetualMarketCreateEvent, @@ -134,7 +132,6 @@ describe('perpetualMarketHandler', () => { LiquidityTiersTable.create(testConstants.defaultLiquidityTier), ]); await liquidityTierRefresher.updateLiquidityTiers(); - await marketRefresher.updateMarkets(); const transactionIndex: number = 0; diff --git a/indexer/services/ender/__tests__/lib/on-message.test.ts b/indexer/services/ender/__tests__/lib/on-message.test.ts index 40b293479e..5a8e287969 100644 --- a/indexer/services/ender/__tests__/lib/on-message.test.ts +++ b/indexer/services/ender/__tests__/lib/on-message.test.ts @@ -6,7 +6,6 @@ import { dbHelpers, IsoString, LiquidityTiersTable, - marketRefresher, MarketTable, perpetualMarketRefresher, PerpetualMarketTable, @@ -893,13 +892,11 @@ describe('on-message', () => { // Initialize assetRefresher await assetRefresher.updateAssets(); await perpetualMarketRefresher.updatePerpetualMarkets(); - await marketRefresher.updateMarkets(); (SubaccountUpdateHandler as jest.Mock).mockReturnValue({ handle: () => { // clear cache so we can confirm that the cache is updated after the error assetRefresher.clear(); perpetualMarketRefresher.clear(); - marketRefresher.clear(); throw new Error(); }, validate: () => null, @@ -909,7 +906,6 @@ describe('on-message', () => { expect(assetRefresher.getAssetsMap()).not.toEqual({}); expect(perpetualMarketRefresher.getPerpetualMarketsMap()).not.toEqual({}); - expect(marketRefresher.getMarketsMap()).not.toEqual({}); }); }); diff --git a/indexer/services/ender/__tests__/validators/asset-validator.test.ts b/indexer/services/ender/__tests__/validators/asset-validator.test.ts index f9ad4d4b51..c093150713 100644 --- a/indexer/services/ender/__tests__/validators/asset-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/asset-validator.test.ts @@ -1,6 +1,6 @@ import { logger } from '@dydxprotocol-indexer/base'; import { AssetCreateEventV1, IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos'; -import { dbHelpers, marketRefresher, testMocks } from '@dydxprotocol-indexer/postgres'; +import { dbHelpers, testMocks } from '@dydxprotocol-indexer/postgres'; import { DydxIndexerSubtypes } from '../../src/lib/types'; import { defaultAssetCreateEvent, defaultHeight, defaultTime, defaultTxHash, @@ -15,7 +15,6 @@ import { AssetValidator } from '../../src/validators/asset-validator'; describe('asset-validator', () => { beforeEach(async () => { await testMocks.seedData(); - await marketRefresher.updateMarkets(); jest.spyOn(logger, 'error'); }); diff --git a/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts b/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts index 8a28a2fd26..8f6a569c43 100644 --- a/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts +++ b/indexer/services/ender/__tests__/validators/perpetual-market-validator.test.ts @@ -1,7 +1,7 @@ import { logger, ParseMessageError } from '@dydxprotocol-indexer/base'; import { PerpetualMarketCreateEventV1, IndexerTendermintBlock, IndexerTendermintEvent } from '@dydxprotocol-indexer/v4-protos'; import { - dbHelpers, marketRefresher, testMocks, perpetualMarketRefresher, + dbHelpers, testMocks, perpetualMarketRefresher, } from '@dydxprotocol-indexer/postgres'; import { DydxIndexerSubtypes } from '../../src/lib/types'; import { @@ -18,7 +18,6 @@ import Long from 'long'; describe('perpetual-market-validator', () => { beforeEach(async () => { await testMocks.seedData(); - await marketRefresher.updateMarkets(); jest.spyOn(logger, 'error'); }); diff --git a/indexer/services/ender/src/caches/block-cache.ts b/indexer/services/ender/src/caches/block-cache.ts index 2690c2486e..3f385d7897 100644 --- a/indexer/services/ender/src/caches/block-cache.ts +++ b/indexer/services/ender/src/caches/block-cache.ts @@ -5,7 +5,6 @@ import { IsolationLevel, Transaction, assetRefresher, - marketRefresher, perpetualMarketRefresher, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; @@ -111,7 +110,6 @@ export async function initializeAllCaches(): Promise { startCandleCache(txId), perpetualMarketRefresher.updatePerpetualMarkets({ txId }), assetRefresher.updateAssets({ txId }), - marketRefresher.updateMarkets({ txId }), ]); await Transaction.rollback(txId); diff --git a/indexer/services/ender/src/handlers/markets/market-create-handler.ts b/indexer/services/ender/src/handlers/markets/market-create-handler.ts index da1d598ce5..2254db9c53 100644 --- a/indexer/services/ender/src/handlers/markets/market-create-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-create-handler.ts @@ -2,7 +2,6 @@ import { logger } from '@dydxprotocol-indexer/base'; import { MarketFromDatabase, MarketModel, - marketRefresher, storeHelpers, } from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; @@ -52,7 +51,6 @@ export class MarketCreateHandler extends Handler { const market: MarketFromDatabase = MarketModel.fromJson( result.rows[0].result.market) as MarketFromDatabase; - marketRefresher.updateMarket(market); return []; } } diff --git a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts index 1276f8eb11..18f381a37f 100644 --- a/indexer/services/ender/src/handlers/markets/market-modify-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-modify-handler.ts @@ -1,6 +1,6 @@ import { logger } from '@dydxprotocol-indexer/base'; import { - MarketFromDatabase, marketRefresher, storeHelpers, MarketModel, + MarketFromDatabase, storeHelpers, MarketModel, } from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; @@ -52,9 +52,6 @@ export class MarketModifyHandler extends Handler { ); }); - const market: MarketFromDatabase = MarketModel.fromJson( - result.rows[0].result.market) as MarketFromDatabase; - marketRefresher.updateMarket(market); return []; } } diff --git a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts index 03ac35cd76..d751ee5833 100644 --- a/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts +++ b/indexer/services/ender/src/handlers/markets/market-price-update-handler.ts @@ -3,7 +3,7 @@ import { MarketFromDatabase, OraclePriceFromDatabase, OraclePriceModel, - MarketMessageContents, storeHelpers, MarketModel, marketRefresher, + MarketMessageContents, storeHelpers, MarketModel, } from '@dydxprotocol-indexer/postgres'; import { MarketEventV1 } from '@dydxprotocol-indexer/v4-protos'; import * as pg from 'pg'; @@ -62,7 +62,6 @@ export class MarketPriceUpdateHandler extends Handler { const oraclePrice: OraclePriceFromDatabase = OraclePriceModel.fromJson( result.rows[0].result.oracle_price) as OraclePriceFromDatabase; - marketRefresher.updateMarket(market); return [ this.generateKafkaEvent( diff --git a/indexer/services/ender/src/index.ts b/indexer/services/ender/src/index.ts index 1bf40aac97..c72d8542ee 100644 --- a/indexer/services/ender/src/index.ts +++ b/indexer/services/ender/src/index.ts @@ -1,7 +1,7 @@ import { logger, startBugsnag, wrapBackgroundTask } from '@dydxprotocol-indexer/base'; import { stopConsumer, startConsumer } from '@dydxprotocol-indexer/kafka'; import { - assetRefresher, perpetualMarketRefresher, marketRefresher, liquidityTierRefresher, + assetRefresher, perpetualMarketRefresher, liquidityTierRefresher, } from '@dydxprotocol-indexer/postgres'; import { initializeAllCaches } from './caches/block-cache'; @@ -24,7 +24,6 @@ async function startKafka(): Promise { await Promise.all([ perpetualMarketRefresher.updatePerpetualMarkets(), assetRefresher.updateAssets(), - marketRefresher.updateMarkets(), liquidityTierRefresher.updateLiquidityTiers(), ]); // Ender does not need to refresh its caches in a loop because Ender is the only service that diff --git a/indexer/services/ender/src/lib/cache-manager.ts b/indexer/services/ender/src/lib/cache-manager.ts index 83957dc123..de80cd2957 100644 --- a/indexer/services/ender/src/lib/cache-manager.ts +++ b/indexer/services/ender/src/lib/cache-manager.ts @@ -2,7 +2,6 @@ import { IsolationLevel, Transaction, assetRefresher, - marketRefresher, perpetualMarketRefresher, } from '@dydxprotocol-indexer/postgres'; @@ -13,7 +12,6 @@ export async function refreshDataCaches(): Promise { await Promise.all([ perpetualMarketRefresher.updatePerpetualMarkets({ txId, readReplica: true }), assetRefresher.updateAssets({ txId, readReplica: true }), - marketRefresher.updateMarkets({ txId, readReplica: true }), ]); await Transaction.rollback(txId);