Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve vault start pnl query. #2664

Merged
merged 4 commits into from
Dec 17, 2024
Merged

Improve vault start pnl query. #2664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3326276
Add cache for initial vault pnl data.
vincentwschau Dec 13, 2024
103bcab
Use view instead of pnl ticks table for latest pnl ticks.
vincentwschau Dec 13, 2024
2fb9b0b
Fix errors.
vincentwschau Dec 17, 2024
02838dd
Fix typo.
vincentwschau Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions indexer/packages/postgres/src/stores/vault-pnl-ticks-view.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,27 @@ export async function getVaultsPnl(

return result.rows;
}

export async function getLatestVaultPnl(): Promise<PnlTicksFromDatabase[]> {
const result: {
rows: PnlTicksFromDatabase[],
} = await knexReadReplica.getConnection().raw(
`
SELECT DISTINCT ON ("subaccountId")
"id",
"subaccountId",
"equity",
"totalPnl",
"netTransfers",
"createdAt",
"blockHeight",
"blockTime"
FROM ${VAULT_HOURLY_PNL_VIEW}
ORDER BY "subaccountId", "blockTime" DESC;
`,
) as unknown as {
rows: PnlTicksFromDatabase[],
};

return result.rows;
}
3 changes: 3 additions & 0 deletions indexer/services/comlink/__tests__/controllers/api/v4/vault-controller.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { getFixedRepresentation, sendRequest } from '../../../helpers/helpers';
import { DateTime, Settings } from 'luxon';
import Big from 'big.js';
import config from '../../../../src/config';
import { clearVaultStartPnl, startVaultStartPnlCache } from '../../../../src/caches/vault-start-pnl';

describe('vault-controller#V4', () => {
const latestBlockHeight: string = '25';
Expand Down Expand Up @@ -131,6 +132,7 @@ describe('vault-controller#V4', () => {
await dbHelpers.clearData();
await VaultPnlTicksView.refreshDailyView();
await VaultPnlTicksView.refreshHourlyView();
clearVaultStartPnl();
config.VAULT_PNL_HISTORY_HOURS = vaultPnlHistoryHoursPrev;
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS = vaultPnlLastPnlWindowPrev;
config.VAULT_PNL_START_DATE = vaultPnlStartDatePrev;
Expand Down Expand Up @@ -653,6 +655,7 @@ describe('vault-controller#V4', () => {
}
await VaultPnlTicksView.refreshDailyView();
await VaultPnlTicksView.refreshHourlyView();
await startVaultStartPnlCache();

return createdTicks;
}
Expand Down
33 changes: 33 additions & 0 deletions indexer/services/comlink/src/caches/vault-start-pnl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { NodeEnv } from '@dydxprotocol-indexer/base';
import {
PnlTicksFromDatabase,
PnlTicksTable,
} from '@dydxprotocol-indexer/postgres';
import _ from 'lodash';

import { getVaultMapping, getVaultPnlStartDate } from '../lib/helpers';
import { VaultMapping } from '../types';

let vaultStartPnl: PnlTicksFromDatabase[] = [];

export async function startVaultStartPnlCache(): Promise<void> {
const vaultMapping: VaultMapping = await getVaultMapping();
vaultStartPnl = await PnlTicksTable.getLatestPnlTick(
_.keys(vaultMapping),
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
);
}

export function getVaultStartPnl(): PnlTicksFromDatabase[] {
return vaultStartPnl;
}

export function clearVaultStartPnl(): void {
if (process.env.NODE_ENV !== NodeEnv.TEST) {
throw Error('cannot clear vault start pnl cache outside of test environment');
}

vaultStartPnl = [];
}
126 changes: 21 additions & 105 deletions indexer/services/comlink/src/controllers/api/v4/vault-controller.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { logger, stats } from '@dydxprotocol-indexer/base';
import { stats } from '@dydxprotocol-indexer/base';
import {
PnlTicksFromDatabase,
PnlTicksTable,
perpetualMarketRefresher,
PerpetualMarketFromDatabase,
USDC_ASSET_ID,
Expand All @@ -22,7 +21,6 @@ import {
BlockFromDatabase,
FundingIndexUpdatesTable,
PnlTickInterval,
VaultTable,
VaultFromDatabase,
MEGAVAULT_SUBACCOUNT_ID,
TransferFromDatabase,
Expand All @@ -42,10 +40,13 @@ import {
} from 'tsoa';

import { getReqRateLimiter } from '../../../caches/rate-limiters';
import { getVaultStartPnl } from '../../../caches/vault-start-pnl';
import config from '../../../config';
import {
aggregateHourlyPnlTicks,
getSubaccountResponse,
getVaultMapping,
getVaultPnlStartDate,
handleControllerError,
} from '../../../lib/helpers';
import { rateLimiterMiddleware } from '../../../lib/rate-limit';
Expand Down Expand Up @@ -108,7 +109,7 @@ class VaultController extends Controller {
getVaultPositions(vaultSubaccounts),
BlockTable.getLatest(),
getMainSubaccountEquity(),
getLatestPnlTick(vaultSubaccountIdsWithMainSubaccount, _.values(vaultSubaccounts)),
getLatestPnlTick(_.values(vaultSubaccounts)),
getFirstMainVaultTransferDateTime(),
]);
stats.timing(
Expand Down Expand Up @@ -162,7 +163,7 @@ class VaultController extends Controller {
getVaultSubaccountPnlTicks(_.keys(vaultSubaccounts), getResolution(resolution)),
getVaultPositions(vaultSubaccounts),
BlockTable.getLatest(),
getLatestPnlTicks(_.keys(vaultSubaccounts)),
getLatestPnlTicks(),
]);
const latestTicksBySubaccountId: Dictionary<PnlTicksFromDatabase> = _.keyBy(
latestTicks,
Expand Down Expand Up @@ -348,27 +349,13 @@ async function getVaultSubaccountPnlTicks(
windowSeconds = config.VAULT_PNL_HISTORY_HOURS * 60 * 60; // hours to seconds
}

const [
pnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
VaultPnlTicksView.getVaultsPnl(
resolution,
windowSeconds,
getVaultPnlStartDate(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
resolution,
windowSeconds,
getVaultPnlStartDate(),
);

return adjustVaultPnlTicks(pnlTicks, adjustByPnlTicks);
return adjustVaultPnlTicks(pnlTicks, getVaultStartPnl());
}

async function getVaultPositions(
Expand Down Expand Up @@ -559,60 +546,26 @@ function getPnlTicksWithCurrentTick(
return pnlTicks.concat([currentTick]);
}

export async function getLatestPnlTicks(
vaultSubaccountIds: string[],
): Promise<PnlTicksFromDatabase[]> {
const [
latestPnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
DateTime.now().toUTC(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
export async function getLatestPnlTicks(): Promise<PnlTicksFromDatabase[]> {
const latestPnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getLatestVaultPnl();
const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
latestPnlTicks,
adjustByPnlTicks,
getVaultStartPnl(),
);
return adjustedPnlTicks;
}

export async function getLatestPnlTick(
vaultSubaccountIds: string[],
vaults: VaultFromDatabase[],
): Promise<PnlTicksFromDatabase | undefined> {
const [
pnlTicks,
adjustByPnlTicks,
] : [
PnlTicksFromDatabase[],
PnlTicksFromDatabase[],
] = await Promise.all([
VaultPnlTicksView.getVaultsPnl(
PnlTickInterval.hour,
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
getVaultPnlStartDate(),
),
PnlTicksTable.getLatestPnlTick(
vaultSubaccountIds,
// Add a buffer of 10 minutes to get the first PnL tick for PnL data as PnL ticks aren't
// created exactly on the hour.
getVaultPnlStartDate().plus({ minutes: 10 }),
),
]);
const pnlTicks: PnlTicksFromDatabase[] = await VaultPnlTicksView.getVaultsPnl(
PnlTickInterval.hour,
config.VAULT_LATEST_PNL_TICK_WINDOW_HOURS * 60 * 60,
getVaultPnlStartDate(),
);
const adjustedPnlTicks: PnlTicksFromDatabase[] = adjustVaultPnlTicks(
pnlTicks,
adjustByPnlTicks,
getVaultStartPnl(),
);
// Aggregate and get pnl tick closest to the hour
const aggregatedTicks: PnlTicksFromDatabase[] = aggregateVaultPnlTicks(
Expand Down Expand Up @@ -802,41 +755,4 @@ function adjustVaultPnlTicks(
});
}

async function getVaultMapping(): Promise<VaultMapping> {
const vaults: VaultFromDatabase[] = await VaultTable.findAll(
{},
[],
{},
);
const vaultMapping: VaultMapping = _.zipObject(
vaults.map((vault: VaultFromDatabase): string => {
return SubaccountTable.uuid(vault.address, 0);
}),
vaults,
);
const validVaultMapping: VaultMapping = {};
for (const subaccountId of _.keys(vaultMapping)) {
const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
.getPerpetualMarketFromClobPairId(
vaultMapping[subaccountId].clobPairId,
);
if (perpetual === undefined) {
logger.warning({
at: 'VaultController#getVaultPositions',
message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
'perpetual market.',
subaccountId,
});
continue;
}
validVaultMapping[subaccountId] = vaultMapping[subaccountId];
}
return validVaultMapping;
}

function getVaultPnlStartDate(): DateTime {
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
return startDate;
}

export default router;
3 changes: 3 additions & 0 deletions indexer/services/comlink/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
} from '@dydxprotocol-indexer/base';
import { perpetualMarketRefresher, liquidityTierRefresher } from '@dydxprotocol-indexer/postgres';

import { startVaultStartPnlCache } from './caches/vault-start-pnl';
import config from './config';
import IndexV4 from './controllers/api/index-v4';
import { connect as connectToRedis } from './helpers/redis/redis-controller';
Expand Down Expand Up @@ -42,6 +43,8 @@ async function start() {
]);
wrapBackgroundTask(perpetualMarketRefresher.start(), true, 'startUpdatePerpetualMarkets');
wrapBackgroundTask(liquidityTierRefresher.start(), true, 'startUpdateLiquidityTiers');
// Initialize cache for vault start PnL
await startVaultStartPnlCache();

await connectToRedis();
logger.info({
Expand Down
41 changes: 41 additions & 0 deletions indexer/services/comlink/src/lib/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
AssetFromDatabase,
AssetColumns,
MarketColumns,
VaultFromDatabase, VaultTable, perpetualMarketRefresher,
} from '@dydxprotocol-indexer/postgres';
import Big from 'big.js';
import express from 'express';
Expand All @@ -47,6 +48,7 @@ import {
PerpetualPositionWithFunding,
Risk,
SubaccountResponseObject,
VaultMapping,
} from '../types';
import { ZERO, ZERO_USDC_POSITION } from './constants';
import { InvalidParamError, NotFoundError } from './errors';
Expand Down Expand Up @@ -720,3 +722,42 @@ export function aggregateHourlyPnlTicks(
};
});
}

/* ------- VAULT HELPERS ------- */

export async function getVaultMapping(): Promise<VaultMapping> {
const vaults: VaultFromDatabase[] = await VaultTable.findAll(
{},
[],
{},
);
const vaultMapping: VaultMapping = _.zipObject(
vaults.map((vault: VaultFromDatabase): string => {
return SubaccountTable.uuid(vault.address, 0);
}),
vaults,
);
const validVaultMapping: VaultMapping = {};
for (const subaccountId of _.keys(vaultMapping)) {
const perpetual: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher
.getPerpetualMarketFromClobPairId(
vaultMapping[subaccountId].clobPairId,
);
if (perpetual === undefined) {
logger.warning({
at: 'get-vault-mapping',
message: `Vault clob pair id ${vaultMapping[subaccountId]} does not correspond to a ` +
'perpetual market.',
subaccountId,
});
continue;
}
validVaultMapping[subaccountId] = vaultMapping[subaccountId];
}
return validVaultMapping;
}

export function getVaultPnlStartDate(): DateTime {
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
return startDate;
}
Comment on lines +760 to +763
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add validation for config value and date format.

The function should validate the config value exists and handle invalid date formats.

Apply this diff:

 export function getVaultPnlStartDate(): DateTime {
+  if (!config.VAULT_PNL_START_DATE) {
+    throw new Error('VAULT_PNL_START_DATE is not configured');
+  }
   const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
+  if (!startDate.isValid) {
+    throw new Error(`Invalid date format in VAULT_PNL_START_DATE: ${config.VAULT_PNL_START_DATE}`);
+  }
   return startDate;
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
export function getVaultPnlStartDate(): DateTime {
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
return startDate;
}
export function getVaultPnlStartDate(): DateTime {
if (!config.VAULT_PNL_START_DATE) {
throw new Error('VAULT_PNL_START_DATE is not configured');
}
const startDate: DateTime = DateTime.fromISO(config.VAULT_PNL_START_DATE).toUTC();
if (!startDate.isValid) {
throw new Error(`Invalid date format in VAULT_PNL_START_DATE: ${config.VAULT_PNL_START_DATE}`);
}
return startDate;
}

5 changes: 5 additions & 0 deletions indexer/services/comlink/src/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
TradeType,
TradingRewardAggregationPeriod,
TransferType,
VaultFromDatabase,
} from '@dydxprotocol-indexer/postgres';
import { RedisOrder } from '@dydxprotocol-indexer/v4-protos';
import Big from 'big.js';
Expand Down Expand Up @@ -691,6 +692,10 @@ export interface MegavaultHistoricalPnlRequest {

export interface VaultsHistoricalPnlRequest extends MegavaultHistoricalPnlRequest {}

export interface VaultMapping {
[subaccountId: string]: VaultFromDatabase,
}

/* ------- Affiliates Types ------- */
export interface AffiliateMetadataRequest{
address: string,
Expand Down
Loading