From b7f2f32e83eb6c5e4c423048025ecac1cd321280 Mon Sep 17 00:00:00 2001 From: Christopher Li Date: Mon, 16 Oct 2023 15:52:00 -0400 Subject: [PATCH] [IND-288]: Remove unused socks fields and add candle resolutions for websockets --- .../src/types/websocket-message-types.ts | 3 +- .../__tests__/lib/candles-generator.test.ts | 4 +- .../ender/src/lib/candles-generator.ts | 2 +- .../helpers/from-kafka-helpers.test.ts | 34 +++++------------ .../socks/src/helpers/from-kafka-helpers.ts | 8 ---- indexer/services/socks/src/types.ts | 38 +------------------ 6 files changed, 15 insertions(+), 74 deletions(-) diff --git a/indexer/packages/postgres/src/types/websocket-message-types.ts b/indexer/packages/postgres/src/types/websocket-message-types.ts index 930e7364b3..51376e1f06 100644 --- a/indexer/packages/postgres/src/types/websocket-message-types.ts +++ b/indexer/packages/postgres/src/types/websocket-message-types.ts @@ -1,3 +1,4 @@ +import { CandleResolution } from './candle-types'; import { FillType, Liquidity } from './fill-types'; import { OrderSide, @@ -226,8 +227,8 @@ export interface OraclePriceMarket { /* ------- CandleMessageContents ------- */ -// Does not include resolution, because resolution is included in CandleMessage export interface CandleMessageContents { + resolution: CandleResolution, startedAt: IsoString, ticker: string, low: string, diff --git a/indexer/services/ender/__tests__/lib/candles-generator.test.ts b/indexer/services/ender/__tests__/lib/candles-generator.test.ts index b57987f51e..590cebf106 100644 --- a/indexer/services/ender/__tests__/lib/candles-generator.test.ts +++ b/indexer/services/ender/__tests__/lib/candles-generator.test.ts @@ -480,7 +480,7 @@ function verifyAllCandlesEqualsKafkaMessages( Partial> = _.chain(expectedCandles) .keyBy(CandleColumns.resolution) .mapValues((candle: CandleFromDatabase) => { - return _.omit(candle, [CandleColumns.id, CandleColumns.resolution]); + return _.omit(candle, [CandleColumns.id]); }) .value(); @@ -516,7 +516,7 @@ function verifyContainsCandlesKafkaMessages( _.forEach(expectedCandles, (expectedCandle: CandleFromDatabase) => { expect( - _.omit(expectedCandle, [CandleColumns.id, CandleColumns.resolution]), + _.omit(expectedCandle, [CandleColumns.id]), ).toEqual(resolutionToContent[expectedCandle.resolution]); }); } diff --git a/indexer/services/ender/src/lib/candles-generator.ts b/indexer/services/ender/src/lib/candles-generator.ts index fd4747b45b..c309b6461e 100644 --- a/indexer/services/ender/src/lib/candles-generator.ts +++ b/indexer/services/ender/src/lib/candles-generator.ts @@ -442,7 +442,7 @@ export class CandlesGenerator { _.forEach(candles, (candle: CandleFromDatabase) => { const candleMessageContents: CandleMessageContents = _.omit( candle, - [CandleColumns.id, CandleColumns.resolution], + [CandleColumns.id], ); const message: CandleMessage = { contents: JSON.stringify(candleMessageContents), diff --git a/indexer/services/socks/__tests__/helpers/from-kafka-helpers.test.ts b/indexer/services/socks/__tests__/helpers/from-kafka-helpers.test.ts index e93ea3fae4..1c2be43da2 100644 --- a/indexer/services/socks/__tests__/helpers/from-kafka-helpers.test.ts +++ b/indexer/services/socks/__tests__/helpers/from-kafka-helpers.test.ts @@ -1,12 +1,8 @@ import { getChannel, getMessageToForward } from '../../src/helpers/from-kafka-helpers'; import { InvalidForwardMessageError, InvalidTopicError } from '../../src/lib/errors'; import { - CandlesChannelMessageToForward, Channel, MessageToForward, - OrderbooksChannelMessageToForward, - SubaccountsChannelMessageToForward, - TradesChannelMessageToForward, WebsocketTopics, } from '../../src/types'; import { @@ -26,7 +22,7 @@ import { import { KafkaMessage } from 'kafkajs'; import { createKafkaMessage } from './kafka'; import { - CandleMessage, CandleMessage_Resolution, + CandleMessage, MarketMessage, OrderbookMessage, SubaccountMessage, @@ -37,7 +33,6 @@ import { dbHelpers, testMocks, perpetualMarketRefresher, - PROTO_TO_CANDLE_RESOLUTION, CandleResolution, } from '@dydxprotocol-indexer/postgres'; @@ -74,37 +69,28 @@ describe('from-kafka-helpers', () => { const message: KafkaMessage = createKafkaMessage( Buffer.from(Uint8Array.from(SubaccountMessage.encode(subaccountMessage).finish())), ); - const messageToForward: SubaccountsChannelMessageToForward = getMessageToForward( + const messageToForward: MessageToForward = getMessageToForward( Channel.V4_ACCOUNTS, message, - ) as SubaccountsChannelMessageToForward; + ); expect(messageToForward.channel).toEqual(Channel.V4_ACCOUNTS); expect(messageToForward.id).toEqual(`${defaultOwner}/${defaultAccNumber}`); expect(messageToForward.contents).toEqual(defaultContents); - expect(messageToForward.blockHeight).toEqual(subaccountMessage.blockHeight); - expect(messageToForward.transactionIndex).toEqual(subaccountMessage.transactionIndex); - expect(messageToForward.eventIndex).toEqual(subaccountMessage.eventIndex); }); it('gets correct MessageToForward for candles message', () => { const message: KafkaMessage = createKafkaMessage( Buffer.from(Uint8Array.from(CandleMessage.encode(candlesMessage).finish())), ); - const messageToForward: CandlesChannelMessageToForward = getMessageToForward( + const messageToForward: MessageToForward = getMessageToForward( Channel.V4_CANDLES, message, - ) as CandlesChannelMessageToForward; + ); expect(messageToForward.channel).toEqual(Channel.V4_CANDLES); expect(messageToForward.id).toEqual(`${btcTicker}/${CandleResolution.ONE_MINUTE}`); expect(messageToForward.contents).toEqual(defaultContents); - expect(messageToForward.clobPairId).toEqual(candlesMessage.clobPairId); - if (candlesMessage.resolution !== CandleMessage_Resolution.UNRECOGNIZED) { - expect(messageToForward.resolution).toEqual( - PROTO_TO_CANDLE_RESOLUTION[candlesMessage.resolution], - ); - } }); it('gets correct MessageToForward for market message', () => { @@ -122,30 +108,28 @@ describe('from-kafka-helpers', () => { const message: KafkaMessage = createKafkaMessage( Buffer.from(Uint8Array.from(OrderbookMessage.encode(orderbookMessage).finish())), ); - const messageToForward: OrderbooksChannelMessageToForward = getMessageToForward( + const messageToForward: MessageToForward = getMessageToForward( Channel.V4_ORDERBOOK, message, - ) as OrderbooksChannelMessageToForward; + ); expect(messageToForward.channel).toEqual(Channel.V4_ORDERBOOK); expect(messageToForward.id).toEqual(btcTicker); expect(messageToForward.contents).toEqual(defaultContents); - expect(messageToForward.clobPairId).toEqual(orderbookMessage.clobPairId); }); it('gets correct MessageToForward for trade message', () => { const message: KafkaMessage = createKafkaMessage( Buffer.from(Uint8Array.from(TradeMessage.encode(tradesMessage).finish())), ); - const messageToForward: TradesChannelMessageToForward = getMessageToForward( + const messageToForward: MessageToForward = getMessageToForward( Channel.V4_TRADES, message, - ) as TradesChannelMessageToForward; + ); expect(messageToForward.channel).toEqual(Channel.V4_TRADES); expect(messageToForward.id).toEqual(btcTicker); expect(messageToForward.contents).toEqual(defaultContents); - expect(messageToForward.blockHeight).toEqual(tradesMessage.blockHeight); }); it('throws InvalidForwardMessageError for empty message', () => { diff --git a/indexer/services/socks/src/helpers/from-kafka-helpers.ts b/indexer/services/socks/src/helpers/from-kafka-helpers.ts index 262bbd6413..ac432050db 100644 --- a/indexer/services/socks/src/helpers/from-kafka-helpers.ts +++ b/indexer/services/socks/src/helpers/from-kafka-helpers.ts @@ -42,9 +42,6 @@ export function getMessageToForward( channel, id: getSubaccountMessageId(subaccountMessage), contents: JSON.parse(subaccountMessage.contents), - blockHeight: subaccountMessage.blockHeight, - transactionIndex: subaccountMessage.transactionIndex, - eventIndex: subaccountMessage.eventIndex, version: subaccountMessage.version, }; } @@ -57,8 +54,6 @@ export function getMessageToForward( channel, id: getCandleMessageId(candleMessage), contents: JSON.parse(candleMessage.contents), - clobPairId: candleMessage.clobPairId, - resolution: PROTO_TO_CANDLE_RESOLUTION[candleMessage.resolution], version: candleMessage.version, }; } @@ -77,7 +72,6 @@ export function getMessageToForward( channel, id: getTickerOrThrow(orderbookMessage.clobPairId), contents: JSON.parse(orderbookMessage.contents), - clobPairId: orderbookMessage.clobPairId, version: orderbookMessage.version, }; } @@ -87,8 +81,6 @@ export function getMessageToForward( channel, id: getTickerOrThrow(tradeMessage.clobPairId), contents: JSON.parse(tradeMessage.contents), - blockHeight: tradeMessage.blockHeight, - clobPairId: tradeMessage.clobPairId, version: tradeMessage.version, }; } diff --git a/indexer/services/socks/src/types.ts b/indexer/services/socks/src/types.ts index 188514c9ff..c4ca479268 100644 --- a/indexer/services/socks/src/types.ts +++ b/indexer/services/socks/src/types.ts @@ -1,6 +1,5 @@ import { IncomingMessage as IncomingMessageHttp } from 'http'; -import { CandleResolution } from '@dydxprotocol-indexer/postgres'; import express from 'express'; import WebSocket from 'ws'; @@ -122,7 +121,7 @@ export interface Connection { disconnect?: NodeJS.Timeout; } -export interface MessageToForwardBase { +export interface MessageToForward { channel: Channel; id: string; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -130,41 +129,6 @@ export interface MessageToForwardBase { version: string; } -export type OrderbooksChannelMessageToForward = MessageToForwardBase & { - channel: Channel.V4_ORDERBOOK; - clobPairId: string; -}; - -export type SubaccountsChannelMessageToForward = MessageToForwardBase & { - channel: Channel.V4_ACCOUNTS; - blockHeight: string; - transactionIndex: number; - eventIndex: number; - subaccountId?: string; -}; - -export type TradesChannelMessageToForward = MessageToForwardBase & { - channel: Channel.V4_TRADES; - blockHeight: string; - clobPairId: string; -}; - -export type MarketsChannelMessageToForward = MessageToForwardBase & { - channel: Channel.V4_MARKETS; -}; - -export type CandlesChannelMessageToForward = MessageToForwardBase & { - channel: Channel.V4_CANDLES; - clobPairId: string; - resolution: CandleResolution; -}; - -export type MessageToForward = OrderbooksChannelMessageToForward -| SubaccountsChannelMessageToForward -| TradesChannelMessageToForward -| MarketsChannelMessageToForward -| CandlesChannelMessageToForward; - export interface ResponseWithBody extends express.Response { body: unknown }