From 0dbc3eeeedc47a60c8ff309c80d8531e087da12f Mon Sep 17 00:00:00 2001 From: Christopher Li Date: Mon, 16 Oct 2023 17:22:35 -0400 Subject: [PATCH] Revert "[IND-412]: Correctly set order status for all order types (#577)" This reverts commit 37f665cc1ff8a9a1dcd043613a6396170334990b. --- indexer/CHANGELOG.md | 25 ++ .../order-fills/liquidation-handler.test.ts | 4 +- .../order-fills/order-handler.test.ts | 245 +----------------- .../abstract-order-fill-handler.ts | 60 +---- .../src/scripts/dydx_get_order_status.sql | 42 +-- .../dydx_order_fill_handler_per_order.sql | 4 +- 6 files changed, 53 insertions(+), 327 deletions(-) create mode 100644 indexer/CHANGELOG.md diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md new file mode 100644 index 0000000000..ba1c28b400 --- /dev/null +++ b/indexer/CHANGELOG.md @@ -0,0 +1,25 @@ +# Changelog + +## [Unreleased] + +### Features + +* [#143](https://github.com/dydxprotocol/v4-chain/pull/143) Add websocket events for perpetual markets. + +### Improvements +* [#552](https://github.com/dydxprotocol/v4-chain/pull/552) Updated Elliptic compliance client block addresses with risk scores equal to the threshold in addition to risk score greater than the threshold. + +* [#469](https://github.com/dydxprotocol/v4-chain/pull/469) Added a reason field to `/screen` endpoint to display a reason for blocking an address. + +### Bug Fixes +* [#579](https://github.com/dydxprotocol/v4-chain/pull/579) Fixed bug where open short-term orders were not being returned in the initial payload when subscribing to the v4_subaccounts channel. + +* [#552](https://github.com/dydxprotocol/v4-chain/pull/552) Fixed bug with Elliptic compliance client where the API key was incorrectly used instead of the API secret to generate the auth headers for the Elliptic request. + +* [#528](https://github.com/dydxprotocol/v4-chain/pull/528) Fixed bug with bulk SQL queries with nullable numeric / string / boolean values. + +* [#496](https://github.com/dydxprotocol/v4-chain/pull/496) Don't geo-block requests made to `comlink` if it's from an internal ip. + +* [#460](https://github.com/dydxprotocol/v4-chain/pull/460/files) Fixed track lag timing to output positive numbers. + +### API Breaking Changes diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index a88f11efdc..1eb3d8d72b 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -237,7 +237,7 @@ describe('LiquidationHandler', () => { goodTilOneof, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, reduceOnly: true, clientMetadata: 0, }); @@ -300,7 +300,7 @@ describe('LiquidationHandler', () => { clobPairId: defaultClobPairId, side: makerOrderProto.side === IndexerOrder_Side.SIDE_BUY ? OrderSide.BUY : OrderSide.SELL, orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.GTT, + timeInForce: TimeInForce.IOC, reduceOnly: true, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index 8fb3468971..2ee036061c 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -22,7 +22,6 @@ import { FillTable, FillType, Liquidity, - OrderFromDatabase, OrderSide, OrderStatus, OrderTable, @@ -281,7 +280,7 @@ describe('OrderHandler', () => { goodTilOneof: takerGoodTilOneof, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, reduceOnly: true, clientMetadata: 0, }); @@ -355,7 +354,7 @@ describe('OrderHandler', () => { clobPairId: defaultClobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(takerOrderProto.side), orderFlags: takerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.GTT, + timeInForce: TimeInForce.IOC, reduceOnly: true, goodTilBlock: protocolTranslations.getGoodTilBlock(takerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(takerOrderProto), @@ -870,7 +869,7 @@ describe('OrderHandler', () => { }, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_FILL_OR_KILL, reduceOnly: false, clientMetadata: 0, }); @@ -955,7 +954,7 @@ describe('OrderHandler', () => { clobPairId: defaultClobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(makerOrderProto.side), orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.GTT, + timeInForce: TimeInForce.FOK, reduceOnly: false, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), @@ -1075,7 +1074,7 @@ describe('OrderHandler', () => { }, clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_FILL_OR_KILL, reduceOnly: false, clientMetadata: 0, }); @@ -1166,7 +1165,7 @@ describe('OrderHandler', () => { clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(makerOrderProto.side), orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.GTT, + timeInForce: TimeInForce.FOK, reduceOnly: false, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), @@ -1360,238 +1359,6 @@ describe('OrderHandler', () => { await expectNoCandles(); }); - it.each([ - [ - 'via knex', - false, - ], - [ - 'via SQL function', - true, - ], - ])('correctly sets status for short term IOC orders (%s)', async ( - _name: string, - useSqlFunction: boolean, - ) => { - config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; - const transactionIndex: number = 0; - const eventIndex: number = 0; - const makerQuantums: number = 100; - const makerSubticks: number = 1_000_000; - - const makerOrderProto: IndexerOrder = createOrder({ - subaccountId: defaultSubaccountId, - clientId: 0, - side: IndexerOrder_Side.SIDE_BUY, - quantums: makerQuantums, - subticks: makerSubticks, - goodTilOneof: { - goodTilBlock: 10, - }, - clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, - orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, - reduceOnly: false, - clientMetadata: 0, - }); - - const takerSubticks: number = 150_000; - const takerQuantums: number = 10; - const takerOrderProto: IndexerOrder = createOrder({ - subaccountId: defaultSubaccountId2, - clientId: 0, - side: IndexerOrder_Side.SIDE_SELL, - quantums: takerQuantums, - subticks: takerSubticks, - goodTilOneof: { - goodTilBlock: 10, - }, - clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, - orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, - reduceOnly: true, - clientMetadata: 0, - }); - - const fillAmount: number = takerQuantums; - const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( - makerOrderProto, - takerOrderProto, - fillAmount, - fillAmount, - fillAmount, - ); - const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ - orderFillEvent, - transactionIndex, - eventIndex, - height: parseInt(defaultHeight, 10), - time: defaultTime, - txHash: defaultTxHash, - }); - - await Promise.all([ - // initial position for subaccount 1 - PerpetualPositionTable.create({ - ...defaultPerpetualPosition, - perpetualId: testConstants.defaultPerpetualMarket3.id, - }), - // initial position for subaccount 2 - PerpetualPositionTable.create({ - ...defaultPerpetualPosition, - subaccountId: testConstants.defaultSubaccountId2, - perpetualId: testConstants.defaultPerpetualMarket3.id, - }), - ]); - - await onMessage(kafkaMessage); - - const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); - const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); - - const [makerOrder, takerOrder]: [ - OrderFromDatabase | undefined, - OrderFromDatabase | undefined - ] = await Promise.all([ - OrderTable.findById(makerOrderId), - OrderTable.findById(takerOrderId), - ]); - - expect(makerOrder).toBeDefined(); - expect(takerOrder).toBeDefined(); - - // maker order is partially filled - expect(makerOrder!.status).toEqual(OrderStatus.CANCELED); - // taker order is fully filled - expect(takerOrder!.status).toEqual(OrderStatus.FILLED); - }); - - it.each([ - [ - 'limit', - 'via knex', - false, - IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, - ], - [ - 'limit', - 'via SQL function', - true, - IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, - ], - [ - 'post-only', - 'via knex', - false, - IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, - ], - [ - 'post-only', - 'via SQL function', - true, - IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, - ], - ])('correctly sets status for short term %s orders (%s)', async ( - _orderType: string, - _name: string, - useSqlFunction: boolean, - timeInForce: IndexerOrder_TimeInForce, - ) => { - config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; - const transactionIndex: number = 0; - const eventIndex: number = 0; - const makerQuantums: number = 100; - const makerSubticks: number = 1_000_000; - - const makerOrderProto: IndexerOrder = createOrder({ - subaccountId: defaultSubaccountId, - clientId: 0, - side: IndexerOrder_Side.SIDE_BUY, - quantums: makerQuantums, - subticks: makerSubticks, - goodTilOneof: { - goodTilBlock: 10, - }, - clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, - orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce, - reduceOnly: false, - clientMetadata: 0, - }); - - const takerSubticks: number = 150_000; - const takerQuantums: number = 100; - const takerOrderProto: IndexerOrder = createOrder({ - subaccountId: defaultSubaccountId2, - clientId: 0, - side: IndexerOrder_Side.SIDE_SELL, - quantums: takerQuantums, - subticks: takerSubticks, - goodTilOneof: { - goodTilBlock: 10, - }, - clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, - orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce, - reduceOnly: true, - clientMetadata: 0, - }); - - const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); - await CanceledOrdersCache.addCanceledOrderId(makerOrderId, Date.now(), redisClient); - - const fillAmount: number = 10; - const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( - makerOrderProto, - takerOrderProto, - fillAmount, - fillAmount, - fillAmount, - ); - const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ - orderFillEvent, - transactionIndex, - eventIndex, - height: parseInt(defaultHeight, 10), - time: defaultTime, - txHash: defaultTxHash, - }); - - await Promise.all([ - // initial position for subaccount 1 - PerpetualPositionTable.create({ - ...defaultPerpetualPosition, - perpetualId: testConstants.defaultPerpetualMarket3.id, - }), - // initial position for subaccount 2 - PerpetualPositionTable.create({ - ...defaultPerpetualPosition, - subaccountId: testConstants.defaultSubaccountId2, - perpetualId: testConstants.defaultPerpetualMarket3.id, - }), - ]); - - await onMessage(kafkaMessage); - - const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); - - const [makerOrder, takerOrder]: [ - OrderFromDatabase | undefined, - OrderFromDatabase | undefined - ] = await Promise.all([ - OrderTable.findById(makerOrderId), - OrderTable.findById(takerOrderId), - ]); - - expect(makerOrder).toBeDefined(); - expect(takerOrder).toBeDefined(); - - // maker order is partially filled, and in CanceledOrdersCache - expect(makerOrder!.status).toEqual(OrderStatus.BEST_EFFORT_CANCELED); - // taker order is partially filled, and not in CanceledOrdersCache - expect(takerOrder!.status).toEqual(OrderStatus.OPEN); - }); - async function expectDefaultOrderAndFillSubaccountKafkaMessages( producerSendMock: jest.SpyInstance, eventId: Buffer, diff --git a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts index 32c36ac9ed..12f8ba7fac 100644 --- a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts @@ -22,12 +22,11 @@ import { SubaccountMessageContents, SubaccountTable, TendermintEventTable, - TimeInForce, TradeMessageContents, UpdatedPerpetualPositionSubaccountKafkaObject, USDC_ASSET_ID, } from '@dydxprotocol-indexer/postgres'; -import { getOrderIdHash, ORDER_FLAG_LONG_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; +import { getOrderIdHash } from '@dydxprotocol-indexer/v4-proto-parser'; import { IndexerOrder, IndexerOrder_Side, @@ -290,11 +289,6 @@ export abstract class AbstractOrderFillHandler extends Handler { return updatedPerpetualPosition; } - /** - * Upsert the an order based on the event processed by the handler - * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation - * orders - */ protected upsertOrderFromEvent( perpetualMarket: PerpetualMarketFromDatabase, order: IndexerOrder, @@ -307,14 +301,6 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilledFromProto.toString(10), perpetualMarket.atomicResolution, ); - const timeInForce: TimeInForce = protocolTranslations.protocolOrderTIFToTIF(order.timeInForce); - const status: OrderStatus = this.getOrderStatus( - isCanceled, - size, - totalFilled, - order.orderId!.orderFlags, - timeInForce, - ); const orderToCreate: OrderCreateObject = { subaccountId: SubaccountTable.subaccountIdToUuid(order.orderId!.subaccountId!), @@ -325,8 +311,8 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilled, price, type: OrderType.LIMIT, // TODO: Add additional order types once we support - status, - timeInForce, + status: this.getOrderStatus(isCanceled, size, totalFilled), + timeInForce: protocolTranslations.protocolOrderTIFToTIF(order.timeInForce), reduceOnly: order.reduceOnly, orderFlags: order.orderId!.orderFlags.toString(), goodTilBlock: protocolTranslations.getGoodTilBlock(order)?.toString(), @@ -339,49 +325,17 @@ export abstract class AbstractOrderFillHandler extends Handler { return OrderTable.upsert(orderToCreate, { txId: this.txId }); } - /** - * The obvious case is if totalFilled >= size, then the order status should always be `FILLED`. - * The difficult case is if totalFilled < size after a fill, then we need to keep the following - * cases in mind: - * 1. Stateful Orders - All cancelations are on-chain events, so the order can be `OPEN` or - * `BEST_EFFORT_CANCELED` if the order is in the CanceledOrdersCache. - * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so - * totalFilled cannot be < size. - * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single - * block, so status should be `CANCELED`. - * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be - * set to `BEST_EFFORT_CANCELED`, otherwise `OPEN`. - * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation - * orders - */ protected getOrderStatus( isCanceled: boolean, size: string, totalFilled: string, - orderFlags: number, - timeInForce: TimeInForce, ): OrderStatus { - if (Big(totalFilled).gte(size)) { - return OrderStatus.FILLED; - } else if (orderFlags === ORDER_FLAG_LONG_TERM) { // 1. Stateful Order - if (isCanceled) { - return OrderStatus.BEST_EFFORT_CANCELED; - } - return OrderStatus.OPEN; - } else if (timeInForce === TimeInForce.FOK) { // 2. Short-term FOK - logger.error({ - at: 'orderFillHandler#getOrderStatus', - message: 'FOK orders should never be partially filled', - blockHeight: this.block.height, - transactionIndex: this.indexerTendermintEvent.transactionIndex, - eventIndex: this.indexerTendermintEvent.eventIndex, - }); - return OrderStatus.CANCELED; - } else if (timeInForce === TimeInForce.IOC) { // 3. Short-term IOC - return OrderStatus.CANCELED; - } else if (isCanceled) { // 4. Short-term Limit & Post-only + if (isCanceled) { return OrderStatus.BEST_EFFORT_CANCELED; } + if (Big(size).lte(totalFilled)) { + return OrderStatus.FILLED; + } return OrderStatus.OPEN; } diff --git a/indexer/services/ender/src/scripts/dydx_get_order_status.sql b/indexer/services/ender/src/scripts/dydx_get_order_status.sql index 354b1f5577..10aed2b394 100644 --- a/indexer/services/ender/src/scripts/dydx_get_order_status.sql +++ b/indexer/services/ender/src/scripts/dydx_get_order_status.sql @@ -1,39 +1,19 @@ /** - Returns the order status given the total filled amount, the order size, whether the order was - cancelled, order flags, and time in force. - * The obvious case is if totalFilled >= size, then the order status should always be `FILLED`. - * The difficult case is if totalFilled < size after a fill, then we need to keep the following - * cases in mind: - * 1. Stateful Orders - All cancelations are on-chain events, so the order can be `OPEN` or - * `BEST_EFFORT_CANCELED` if the order is in the CanceledOrdersCache. - * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so - * totalFilled cannot be < size. - * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single - * block, so status should be `CANCELED`. - * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be - * set to `BEST_EFFORT_CANCELED`, otherwise `OPEN`. + Returns the order status given the total filled amount, the order size and whether the order was cancelled. */ -CREATE OR REPLACE FUNCTION get_order_status(total_filled numeric, size numeric, is_cancelled boolean, order_flags bigint, time_in_force text) +CREATE OR REPLACE FUNCTION get_order_status(total_filled numeric, size numeric, is_cancelled boolean) RETURNS text AS $$ +DECLARE + order_status text; BEGIN - IF total_filled >= size THEN - RETURN 'FILLED'; - /** Order flag of 64 is a stateful term order */ - ELSIF order_flags = 64 THEN /** 1. Stateful Order */ - IF is_cancelled THEN - RETURN 'BEST_EFFORT_CANCELED'; - ELSE - RETURN 'OPEN'; - END IF; - ELSIF time_in_force = 'FOK' THEN /** 2. Short-term FOK */ - /** TODO(IND-439): Match knex and SQL behavior, have this log and return CANCELED */ - RAISE EXCEPTION 'FOK orders should never be partially filled'; - ELSIF time_in_force = 'IOC' THEN /** 3. Short-term IOC */ - RETURN 'CANCELED'; - ELSIF is_cancelled THEN /** 4. Short-term Limit & Postonly */ - RETURN 'BEST_EFFORT_CANCELED'; + IF is_cancelled = true THEN + order_status = 'BEST_EFFORT_CANCELED'; + ELSIF total_filled >= size THEN + order_status = 'FILLED'; ELSE - RETURN 'OPEN'; + order_status = 'OPEN'; END IF; + + RETURN order_status; END; $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; diff --git a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql index 402a068075..20d8248af7 100644 --- a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql @@ -99,7 +99,7 @@ BEGIN IF FOUND THEN order_record."totalFilled" = total_filled; - order_record."status" = get_order_status(total_filled, order_record.size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = get_order_status(total_filled, order_record.size, is_cancelled); UPDATE orders SET @@ -125,7 +125,7 @@ BEGIN order_record."type" = 'LIMIT'; /* TODO: Add additional order types once we support */ order_record."totalFilled" = fill_amount; - order_record."status" = get_order_status(fill_amount, order_size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); + order_record."status" = get_order_status(fill_amount, order_size, is_cancelled); order_record."createdAtHeight" = block_height; order_record."updatedAt" = block_time; order_record."updatedAtHeight" = block_height;