From 33262768a0fd802fbc345d191f74f4812e7a13b1 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:58:24 -0500 Subject: [PATCH 1/4] Add cache for initial vault pnl data. --- .../api/v4/vault-controller.test.ts | 3 + .../comlink/src/caches/vault-start-pnl.ts | 32 +++++ .../controllers/api/v4/vault-controller.ts | 117 +++--------------- indexer/services/comlink/src/index.ts | 3 + indexer/services/comlink/src/lib/helpers.ts | 43 +++++++ indexer/services/comlink/src/types.ts | 5 + 6 files changed, 106 insertions(+), 97 deletions(-) create mode 100644 indexer/services/comlink/src/caches/vault-start-pnl.ts diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 4035552057..1f47fa5339 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -24,6 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; import { DateTime, Settings } from 'luxon'; import Big from 'big.js'; import config from '../../../../src/config'; +import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl' describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; @@ -131,6 +132,7 @@ describe('vault-controller#V4', () => { await dbHelpers.clearData(); await VaultPnlTicksView.refreshDailyView(); await VaultPnlTicksView.refreshHourlyView(); + clearVaultStartPnl(); config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev; config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev; config.VAULT_PNL_START_DATE = vaultPnlStartDatePrev; @@ -653,6 +655,7 @@ describe('vault-controller#V4', () => { } await VaultPnlTicksView.refreshDailyView(); await VaultPnlTicksView.refreshHourlyView(); + await startVaultStartPnlCache(); return createdTicks; } diff --git a/indexer/services/comlink/src/caches/vault-start-pnl.ts b/indexer/services/comlink/src/caches/vault-start-pnl.ts new file mode 100644 index 0000000000..2214db0ac2 --- /dev/null +++ b/indexer/services/comlink/src/caches/vault-start-pnl.ts @@ -0,0 +1,32 @@ +import { + PnlTicksFromDatabase, + PnlTicksTable, +} from '@dydxprotocol-indexer/postgres'; +import _ from 'lodash'; +import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers'; +import { VaultMapping } from '../types'; +import { NodeEnv } from '@dydxprotocol-indexer/base'; + +let vaultStartPnl: PnlTicksFromDatabase[] = []; + +export async function startVaultStartPnlCache(): Promise { + const vaultMapping: VaultMapping = await getVaultMapping(); + vaultStartPnl = await PnlTicksTable.getLatestPnlTick( + _.keys(vaultMapping), + // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't + // created exactly on the hour. + getVaultPnlStartDate().plus({ minutes: 10 }), + ); +} + +export function getVaultStartPnl(): PnlTicksFromDatabase[] { + return vaultStartPnl; +} + +export function clearVaultStartPnl(): void { + if (process.env.NODE_ENV !== NodeEnv.TEST) { + throw Error('cannot clear vault start pnl cache outside of test environment'); + } + + vaultStartPnl = []; +} diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 4df7f35fe4..2ca3efe6f3 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -22,7 +22,6 @@ import { BlockFromDatabase, FundingIndexUpdatesTable, PnlTickInterval, - VaultTable, VaultFromDatabase, MEGAVAULT_SUBACCOUNT_ID, TransferFromDatabase, @@ -46,6 +45,8 @@ import config from '../../../config'; import { aggregateHourlyPnlTicks, getSubaccountResponse, + getVaultMapping, + getVaultPnlStartDate, handleControllerError, } from '../../../lib/helpers'; import { rateLimiterMiddleware } from '../../../lib/rate-limit'; @@ -64,6 +65,7 @@ import { VaultsHistoricalPnlRequest, AggregatedPnlTick, } from '../../../types'; +import { getVaultStartPnl } from '../../../caches/vault-start-pnl'; const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; @@ -348,27 +350,13 @@ async function getVaultSubaccountPnlTicks( windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds } - const [ - pnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - VaultPnlTicksView.getVaultsPnl( - resolution, - windowSeconds, - getVaultPnlStartDate(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); + const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + resolution, + windowSeconds, + getVaultPnlStartDate(), + ); - return adjustVaultPnlTicks(pnlTicks, adjustByPnlTicks); + return adjustVaultPnlTicks(pnlTicks, getVaultStartPnl()); } async function getVaultPositions( @@ -562,27 +550,13 @@ function getPnlTicksWithCurrentTick( export async function getLatestPnlTicks( vaultSubaccountIds: string[], ): Promise { - const [ - latestPnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - DateTime.now().toUTC(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); + const latestPnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getLatestPnlTick( + vaultSubaccountIds, + DateTime.now().toUTC(), + ); const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks( latestPnlTicks, - adjustByPnlTicks, + getVaultStartPnl(), ); return adjustedPnlTicks; } @@ -591,28 +565,14 @@ export async function getLatestPnlTick( vaultSubaccountIds: string[], vaults: VaultFromDatabase[], ): Promise { - const [ - pnlTicks, - adjustByPnlTicks, - ] : [ - PnlTicksFromDatabase[], - PnlTicksFromDatabase[], - ] = await Promise.all([ - VaultPnlTicksView.getVaultsPnl( - PnlTickInterval.hour, - config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60, - getVaultPnlStartDate(), - ), - PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - // Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't - // created exactly on the hour. - getVaultPnlStartDate().plus({ minutes: 10 }), - ), - ]); + const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( + PnlTickInterval.hour, + config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60, + getVaultPnlStartDate(), + ); const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks( pnlTicks, - adjustByPnlTicks, + getVaultStartPnl(), ); // Aggregate and get pnl tick closest to the hour const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks( @@ -802,41 +762,4 @@ function adjustVaultPnlTicks( }); } -async function getVaultMapping(): Promise { - const vaults: VaultFromDatabase[] = await VaultTable.findAll( - {}, - [], - {}, - ); - const vaultMapping: VaultMapping = _.zipObject( - vaults.map((vault: VaultFromDatabase): string => { - return SubaccountTable.uuid(vault.address, 0); - }), - vaults, - ); - const validVaultMapping: VaultMapping = {}; - for (const subaccountId of _.keys(vaultMapping)) { - const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId( - vaultMapping[subaccountId].clobPairId, - ); - if (perpetual === undefined) { - logger.warning({ - at: 'VaultController#getVaultPositions', - message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + - 'perpetual market.', - subaccountId, - }); - continue; - } - validVaultMapping[subaccountId] = vaultMapping[subaccountId]; - } - return validVaultMapping; -} - -function getVaultPnlStartDate(): DateTime { - const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC(); - return startDate; -} - export default router; diff --git a/indexer/services/comlink/src/index.ts b/indexer/services/comlink/src/index.ts index 584fe8edff..9759dada12 100644 --- a/indexer/services/comlink/src/index.ts +++ b/indexer/services/comlink/src/index.ts @@ -9,6 +9,7 @@ import config from './config'; import IndexV4 from './controllers/api/index-v4'; import { connect as connectToRedis } from './helpers/redis/redis-controller'; import Server from './request-helpers/server'; +import { startVaultStartPnlCache } from './caches/vault-start-pnl'; process.on('SIGTERM', () => { logger.info({ @@ -42,6 +43,8 @@ async function start() { ]); wrapBackgroundTask(perpetualMarketRefresher.start(), true, 'startUpdatePerpetualMarkets'); wrapBackgroundTask(liquidityTierRefresher.start(), true, 'startUpdateLiquidityTiers'); + // Initialize cache for vault start PnL + await startVaultStartPnlCache(); await connectToRedis(); logger.info({ diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index b072ca4b0d..08f1293b81 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -47,9 +47,13 @@ import { PerpetualPositionWithFunding, Risk, SubaccountResponseObject, + VaultMapping, } from '../types'; import { ZERO, ZERO_USDC_POSITION } from './constants'; import { InvalidParamError, NotFoundError } from './errors'; +import { VaultFromDatabase } from '@dydxprotocol-indexer/postgres'; +import { VaultTable } from '@dydxprotocol-indexer/postgres'; +import { perpetualMarketRefresher } from '@dydxprotocol-indexer/postgres'; /* ------- GENERIC HELPERS ------- */ @@ -720,3 +724,42 @@ export function aggregateHourlyPnlTicks( }; }); } + +/* ------- VAULT HELPERS ------- */ + +export async function getVaultMapping(): Promise { + const vaults: VaultFromDatabase[] = await VaultTable.findAll( + {}, + [], + {}, + ); + const vaultMapping: VaultMapping = _.zipObject( + vaults.map((vault: VaultFromDatabase): string => { + return SubaccountTable.uuid(vault.address, 0); + }), + vaults, + ); + const validVaultMapping: VaultMapping = {}; + for (const subaccountId of _.keys(vaultMapping)) { + const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId( + vaultMapping[subaccountId].clobPairId, + ); + if (perpetual === undefined) { + logger.warning({ + at: 'VaultController#getVaultPositions', + message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + + 'perpetual market.', + subaccountId, + }); + continue; + } + validVaultMapping[subaccountId] = vaultMapping[subaccountId]; + } + return validVaultMapping; +} + +export function getVaultPnlStartDate(): DateTime { + const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC(); + return startDate; +} diff --git a/indexer/services/comlink/src/types.ts b/indexer/services/comlink/src/types.ts index fc83a92e29..30e540d4b0 100644 --- a/indexer/services/comlink/src/types.ts +++ b/indexer/services/comlink/src/types.ts @@ -24,6 +24,7 @@ import { TradeType, TradingRewardAggregationPeriod, TransferType, + VaultFromDatabase, } from '@dydxprotocol-indexer/postgres'; import { RedisOrder } from '@dydxprotocol-indexer/v4-protos'; import Big from 'big.js'; @@ -691,6 +692,10 @@ export interface MegavaultHistoricalPnlRequest { export interface VaultsHistoricalPnlRequest extends MegavaultHistoricalPnlRequest {} +export interface VaultMapping { + [subaccountId: string]: VaultFromDatabase, +} + /* ------- Affiliates Types ------- */ export interface AffiliateMetadataRequest{ address: string, From 103bcab65354514f34a645d2f917f5b306d37e99 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:49:12 -0500 Subject: [PATCH 2/4] Use view instead of pnl ticks table for latest pnl ticks. --- .../src/stores/vault-pnl-ticks-view.ts | 24 +++++++++++++++++++ .../api/v4/vault-controller.test.ts | 2 +- .../comlink/src/caches/vault-start-pnl.ts | 3 ++- .../controllers/api/v4/vault-controller.ts | 14 ++++------- indexer/services/comlink/src/index.ts | 2 +- indexer/services/comlink/src/lib/helpers.ts | 4 +--- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts b/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts index abac0a88b2..1cfa51e46f 100644 --- a/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts +++ b/indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts @@ -62,3 +62,27 @@ export async function getVaultsPnl( return result.rows; } + +export async function getLatestVaultPnl(): Promise { + const result: { + rows: PnlTicksFromDatabase[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT DISTINCT ON ("subaccountId") + "id", + "subaccountId", + "equity", + "totalPnl", + "netTransfers", + "createdAt", + "blockHeight", + "blockTime" + FROM ${VAULT_HOURLY_PNL_VIEW} + ORDER BY "subaccountId", "blockTime" DESC; + `, + ) as unknown as { + rows: PnlTicksFromDatabase[], + }; + + return result.rows; +} diff --git a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts index 1f47fa5339..03a453daff 100644 --- a/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts +++ b/indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts @@ -24,7 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers'; import { DateTime, Settings } from 'luxon'; import Big from 'big.js'; import config from '../../../../src/config'; -import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl' +import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl'; describe('vault-controller#V4', () => { const latestBlockHeight: string = '25'; diff --git a/indexer/services/comlink/src/caches/vault-start-pnl.ts b/indexer/services/comlink/src/caches/vault-start-pnl.ts index 2214db0ac2..44e25c5d63 100644 --- a/indexer/services/comlink/src/caches/vault-start-pnl.ts +++ b/indexer/services/comlink/src/caches/vault-start-pnl.ts @@ -1,11 +1,12 @@ +import { NodeEnv } from '@dydxprotocol-indexer/base'; import { PnlTicksFromDatabase, PnlTicksTable, } from '@dydxprotocol-indexer/postgres'; import _ from 'lodash'; + import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers'; import { VaultMapping } from '../types'; -import { NodeEnv } from '@dydxprotocol-indexer/base'; let vaultStartPnl: PnlTicksFromDatabase[] = []; diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index 2ca3efe6f3..ac182020db 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -1,7 +1,6 @@ -import { logger, stats } from '@dydxprotocol-indexer/base'; +import { stats } from '@dydxprotocol-indexer/base'; import { PnlTicksFromDatabase, - PnlTicksTable, perpetualMarketRefresher, PerpetualMarketFromDatabase, USDC_ASSET_ID, @@ -41,6 +40,7 @@ import { } from 'tsoa'; import { getReqRateLimiter } from '../../../caches/rate-limiters'; +import { getVaultStartPnl } from '../../../caches/vault-start-pnl'; import config from '../../../config'; import { aggregateHourlyPnlTicks, @@ -65,7 +65,6 @@ import { VaultsHistoricalPnlRequest, AggregatedPnlTick, } from '../../../types'; -import { getVaultStartPnl } from '../../../caches/vault-start-pnl'; const router: express.Router = express.Router(); const controllerName: string = 'vault-controller'; @@ -547,13 +546,8 @@ function getPnlTicksWithCurrentTick( return pnlTicks.concat([currentTick]); } -export async function getLatestPnlTicks( - vaultSubaccountIds: string[], -): Promise { - const latestPnlTicks: PnlTicksFromDatabase[] = await PnlTicksTable.getLatestPnlTick( - vaultSubaccountIds, - DateTime.now().toUTC(), - ); +export async function getLatestPnlTicks(): Promise { + const latestPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getLatestVaultPnl(); const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks( latestPnlTicks, getVaultStartPnl(), diff --git a/indexer/services/comlink/src/index.ts b/indexer/services/comlink/src/index.ts index 9759dada12..114fb5a713 100644 --- a/indexer/services/comlink/src/index.ts +++ b/indexer/services/comlink/src/index.ts @@ -5,11 +5,11 @@ import { } from '@dydxprotocol-indexer/base'; import { perpetualMarketRefresher, liquidityTierRefresher } from '@dydxprotocol-indexer/postgres'; +import { startVaultStartPnlCache } from './caches/vault-start-pnl'; import config from './config'; import IndexV4 from './controllers/api/index-v4'; import { connect as connectToRedis } from './helpers/redis/redis-controller'; import Server from './request-helpers/server'; -import { startVaultStartPnlCache } from './caches/vault-start-pnl'; process.on('SIGTERM', () => { logger.info({ diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index 08f1293b81..b1d4bc59d8 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -24,6 +24,7 @@ import { AssetFromDatabase, AssetColumns, MarketColumns, + VaultFromDatabase, VaultTable, perpetualMarketRefresher, } from '@dydxprotocol-indexer/postgres'; import Big from 'big.js'; import express from 'express'; @@ -51,9 +52,6 @@ import { } from '../types'; import { ZERO, ZERO_USDC_POSITION } from './constants'; import { InvalidParamError, NotFoundError } from './errors'; -import { VaultFromDatabase } from '@dydxprotocol-indexer/postgres'; -import { VaultTable } from '@dydxprotocol-indexer/postgres'; -import { perpetualMarketRefresher } from '@dydxprotocol-indexer/postgres'; /* ------- GENERIC HELPERS ------- */ From 2fb9b0b05564f2bd3dfe88ca010af85d052f62f3 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:07:44 -0500 Subject: [PATCH 3/4] Fix errors. --- .../comlink/src/controllers/api/v4/vault-controller.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index ac182020db..ee77fe4ec8 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -109,7 +109,7 @@ class VaultController extends Controller { getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), getMainSubaccountEquity(), - getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)), + getLatestPnlTick(_.values(vaultSubaccounts)), getFirstMainVaultTransferDateTime(), ]); stats.timing( @@ -163,7 +163,7 @@ class VaultController extends Controller { getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)), getVaultPositions(vaultSubaccounts), BlockTable.getLatest(), - getLatestPnlTicks(_.keys(vaultSubaccounts)), + getLatestPnlTicks(), ]); const latestTicksBySubaccountId: Dictionary = _.keyBy( latestTicks, @@ -556,7 +556,6 @@ export async function getLatestPnlTicks(): Promise { } export async function getLatestPnlTick( - vaultSubaccountIds: string[], vaults: VaultFromDatabase[], ): Promise { const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl( From 02838dde1dd54a0e61bddada9c89c07e81b05d82 Mon Sep 17 00:00:00 2001 From: Vincent Chau <99756290+vincentwschau@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:38:02 -0500 Subject: [PATCH 4/4] Fix typo. --- indexer/services/comlink/src/lib/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/services/comlink/src/lib/helpers.ts b/indexer/services/comlink/src/lib/helpers.ts index b1d4bc59d8..97ee94915d 100644 --- a/indexer/services/comlink/src/lib/helpers.ts +++ b/indexer/services/comlink/src/lib/helpers.ts @@ -745,7 +745,7 @@ export async function getVaultMapping(): Promise { ); if (perpetual === undefined) { logger.warning({ - at: 'VaultController#getVaultPositions', + at: 'get-vault-mapping', message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` + 'perpetual market.', subaccountId,