diff --git a/indexer/CHANGELOG.md b/indexer/CHANGELOG.md index ba1c28b400..9524059033 100644 --- a/indexer/CHANGELOG.md +++ b/indexer/CHANGELOG.md @@ -7,6 +7,8 @@ * [#143](https://github.com/dydxprotocol/v4-chain/pull/143) Add websocket events for perpetual markets. ### Improvements +* [#577](https://github.com/dydxprotocol/v4-chain/pull/577) Correctly set order status for all order types + * [#552](https://github.com/dydxprotocol/v4-chain/pull/552) Updated Elliptic compliance client block addresses with risk scores equal to the threshold in addition to risk score greater than the threshold. * [#469](https://github.com/dydxprotocol/v4-chain/pull/469) Added a reason field to `/screen` endpoint to display a reason for blocking an address. diff --git a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts index 1eb3d8d72b..a88f11efdc 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/liquidation-handler.test.ts @@ -237,7 +237,7 @@ describe('LiquidationHandler', () => { goodTilOneof, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, reduceOnly: true, clientMetadata: 0, }); @@ -300,7 +300,7 @@ describe('LiquidationHandler', () => { clobPairId: defaultClobPairId, side: makerOrderProto.side === IndexerOrder_Side.SIDE_BUY ? OrderSide.BUY : OrderSide.SELL, orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.IOC, + timeInForce: TimeInForce.GTT, reduceOnly: true, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), diff --git a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts index 2ee036061c..8fb3468971 100644 --- a/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts +++ b/indexer/services/ender/__tests__/handlers/order-fills/order-handler.test.ts @@ -22,6 +22,7 @@ import { FillTable, FillType, Liquidity, + OrderFromDatabase, OrderSide, OrderStatus, OrderTable, @@ -280,7 +281,7 @@ describe('OrderHandler', () => { goodTilOneof: takerGoodTilOneof, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, reduceOnly: true, clientMetadata: 0, }); @@ -354,7 +355,7 @@ describe('OrderHandler', () => { clobPairId: defaultClobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(takerOrderProto.side), orderFlags: takerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.IOC, + timeInForce: TimeInForce.GTT, reduceOnly: true, goodTilBlock: protocolTranslations.getGoodTilBlock(takerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(takerOrderProto), @@ -869,7 +870,7 @@ describe('OrderHandler', () => { }, clobPairId: defaultClobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_FILL_OR_KILL, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, reduceOnly: false, clientMetadata: 0, }); @@ -954,7 +955,7 @@ describe('OrderHandler', () => { clobPairId: defaultClobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(makerOrderProto.side), orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.FOK, + timeInForce: TimeInForce.GTT, reduceOnly: false, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), @@ -1074,7 +1075,7 @@ describe('OrderHandler', () => { }, clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, orderFlags: ORDER_FLAG_SHORT_TERM.toString(), - timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_FILL_OR_KILL, + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, reduceOnly: false, clientMetadata: 0, }); @@ -1165,7 +1166,7 @@ describe('OrderHandler', () => { clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, side: protocolTranslations.protocolOrderSideToOrderSide(makerOrderProto.side), orderFlags: makerOrderProto.orderId!.orderFlags.toString(), - timeInForce: TimeInForce.FOK, + timeInForce: TimeInForce.GTT, reduceOnly: false, goodTilBlock: protocolTranslations.getGoodTilBlock(makerOrderProto)?.toString(), goodTilBlockTime: protocolTranslations.getGoodTilBlockTime(makerOrderProto), @@ -1359,6 +1360,238 @@ describe('OrderHandler', () => { await expectNoCandles(); }); + it.each([ + [ + 'via knex', + false, + ], + [ + 'via SQL function', + true, + ], + ])('correctly sets status for short term IOC orders (%s)', async ( + _name: string, + useSqlFunction: boolean, + ) => { + config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; + const transactionIndex: number = 0; + const eventIndex: number = 0; + const makerQuantums: number = 100; + const makerSubticks: number = 1_000_000; + + const makerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId, + clientId: 0, + side: IndexerOrder_Side.SIDE_BUY, + quantums: makerQuantums, + subticks: makerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + reduceOnly: false, + clientMetadata: 0, + }); + + const takerSubticks: number = 150_000; + const takerQuantums: number = 10; + const takerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId2, + clientId: 0, + side: IndexerOrder_Side.SIDE_SELL, + quantums: takerQuantums, + subticks: takerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce: IndexerOrder_TimeInForce.TIME_IN_FORCE_IOC, + reduceOnly: true, + clientMetadata: 0, + }); + + const fillAmount: number = takerQuantums; + const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( + makerOrderProto, + takerOrderProto, + fillAmount, + fillAmount, + fillAmount, + ); + const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ + orderFillEvent, + transactionIndex, + eventIndex, + height: parseInt(defaultHeight, 10), + time: defaultTime, + txHash: defaultTxHash, + }); + + await Promise.all([ + // initial position for subaccount 1 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + // initial position for subaccount 2 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + subaccountId: testConstants.defaultSubaccountId2, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + ]); + + await onMessage(kafkaMessage); + + const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); + const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); + + const [makerOrder, takerOrder]: [ + OrderFromDatabase | undefined, + OrderFromDatabase | undefined + ] = await Promise.all([ + OrderTable.findById(makerOrderId), + OrderTable.findById(takerOrderId), + ]); + + expect(makerOrder).toBeDefined(); + expect(takerOrder).toBeDefined(); + + // maker order is partially filled + expect(makerOrder!.status).toEqual(OrderStatus.CANCELED); + // taker order is fully filled + expect(takerOrder!.status).toEqual(OrderStatus.FILLED); + }); + + it.each([ + [ + 'limit', + 'via knex', + false, + IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + ], + [ + 'limit', + 'via SQL function', + true, + IndexerOrder_TimeInForce.TIME_IN_FORCE_UNSPECIFIED, + ], + [ + 'post-only', + 'via knex', + false, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + ], + [ + 'post-only', + 'via SQL function', + true, + IndexerOrder_TimeInForce.TIME_IN_FORCE_POST_ONLY, + ], + ])('correctly sets status for short term %s orders (%s)', async ( + _orderType: string, + _name: string, + useSqlFunction: boolean, + timeInForce: IndexerOrder_TimeInForce, + ) => { + config.USE_ORDER_HANDLER_SQL_FUNCTION = useSqlFunction; + const transactionIndex: number = 0; + const eventIndex: number = 0; + const makerQuantums: number = 100; + const makerSubticks: number = 1_000_000; + + const makerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId, + clientId: 0, + side: IndexerOrder_Side.SIDE_BUY, + quantums: makerQuantums, + subticks: makerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce, + reduceOnly: false, + clientMetadata: 0, + }); + + const takerSubticks: number = 150_000; + const takerQuantums: number = 100; + const takerOrderProto: IndexerOrder = createOrder({ + subaccountId: defaultSubaccountId2, + clientId: 0, + side: IndexerOrder_Side.SIDE_SELL, + quantums: takerQuantums, + subticks: takerSubticks, + goodTilOneof: { + goodTilBlock: 10, + }, + clobPairId: testConstants.defaultPerpetualMarket3.clobPairId, + orderFlags: ORDER_FLAG_SHORT_TERM.toString(), + timeInForce, + reduceOnly: true, + clientMetadata: 0, + }); + + const makerOrderId: string = OrderTable.orderIdToUuid(makerOrderProto.orderId!); + await CanceledOrdersCache.addCanceledOrderId(makerOrderId, Date.now(), redisClient); + + const fillAmount: number = 10; + const orderFillEvent: OrderFillEventV1 = createOrderFillEvent( + makerOrderProto, + takerOrderProto, + fillAmount, + fillAmount, + fillAmount, + ); + const kafkaMessage: KafkaMessage = createKafkaMessageFromOrderFillEvent({ + orderFillEvent, + transactionIndex, + eventIndex, + height: parseInt(defaultHeight, 10), + time: defaultTime, + txHash: defaultTxHash, + }); + + await Promise.all([ + // initial position for subaccount 1 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + // initial position for subaccount 2 + PerpetualPositionTable.create({ + ...defaultPerpetualPosition, + subaccountId: testConstants.defaultSubaccountId2, + perpetualId: testConstants.defaultPerpetualMarket3.id, + }), + ]); + + await onMessage(kafkaMessage); + + const takerOrderId: string = OrderTable.orderIdToUuid(takerOrderProto.orderId!); + + const [makerOrder, takerOrder]: [ + OrderFromDatabase | undefined, + OrderFromDatabase | undefined + ] = await Promise.all([ + OrderTable.findById(makerOrderId), + OrderTable.findById(takerOrderId), + ]); + + expect(makerOrder).toBeDefined(); + expect(takerOrder).toBeDefined(); + + // maker order is partially filled, and in CanceledOrdersCache + expect(makerOrder!.status).toEqual(OrderStatus.BEST_EFFORT_CANCELED); + // taker order is partially filled, and not in CanceledOrdersCache + expect(takerOrder!.status).toEqual(OrderStatus.OPEN); + }); + async function expectDefaultOrderAndFillSubaccountKafkaMessages( producerSendMock: jest.SpyInstance, eventId: Buffer, diff --git a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts index 12f8ba7fac..32c36ac9ed 100644 --- a/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts +++ b/indexer/services/ender/src/handlers/order-fills/abstract-order-fill-handler.ts @@ -22,11 +22,12 @@ import { SubaccountMessageContents, SubaccountTable, TendermintEventTable, + TimeInForce, TradeMessageContents, UpdatedPerpetualPositionSubaccountKafkaObject, USDC_ASSET_ID, } from '@dydxprotocol-indexer/postgres'; -import { getOrderIdHash } from '@dydxprotocol-indexer/v4-proto-parser'; +import { getOrderIdHash, ORDER_FLAG_LONG_TERM } from '@dydxprotocol-indexer/v4-proto-parser'; import { IndexerOrder, IndexerOrder_Side, @@ -289,6 +290,11 @@ export abstract class AbstractOrderFillHandler extends Handler { return updatedPerpetualPosition; } + /** + * Upsert the an order based on the event processed by the handler + * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation + * orders + */ protected upsertOrderFromEvent( perpetualMarket: PerpetualMarketFromDatabase, order: IndexerOrder, @@ -301,6 +307,14 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilledFromProto.toString(10), perpetualMarket.atomicResolution, ); + const timeInForce: TimeInForce = protocolTranslations.protocolOrderTIFToTIF(order.timeInForce); + const status: OrderStatus = this.getOrderStatus( + isCanceled, + size, + totalFilled, + order.orderId!.orderFlags, + timeInForce, + ); const orderToCreate: OrderCreateObject = { subaccountId: SubaccountTable.subaccountIdToUuid(order.orderId!.subaccountId!), @@ -311,8 +325,8 @@ export abstract class AbstractOrderFillHandler extends Handler { totalFilled, price, type: OrderType.LIMIT, // TODO: Add additional order types once we support - status: this.getOrderStatus(isCanceled, size, totalFilled), - timeInForce: protocolTranslations.protocolOrderTIFToTIF(order.timeInForce), + status, + timeInForce, reduceOnly: order.reduceOnly, orderFlags: order.orderId!.orderFlags.toString(), goodTilBlock: protocolTranslations.getGoodTilBlock(order)?.toString(), @@ -325,16 +339,48 @@ export abstract class AbstractOrderFillHandler extends Handler { return OrderTable.upsert(orderToCreate, { txId: this.txId }); } + /** + * The obvious case is if totalFilled >= size, then the order status should always be `FILLED`. + * The difficult case is if totalFilled < size after a fill, then we need to keep the following + * cases in mind: + * 1. Stateful Orders - All cancelations are on-chain events, so the order can be `OPEN` or + * `BEST_EFFORT_CANCELED` if the order is in the CanceledOrdersCache. + * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so + * totalFilled cannot be < size. + * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single + * block, so status should be `CANCELED`. + * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be + * set to `BEST_EFFORT_CANCELED`, otherwise `OPEN`. + * @param isCanceled - if the order is in the CanceledOrderCache, always false for liquidiation + * orders + */ protected getOrderStatus( isCanceled: boolean, size: string, totalFilled: string, + orderFlags: number, + timeInForce: TimeInForce, ): OrderStatus { - if (isCanceled) { - return OrderStatus.BEST_EFFORT_CANCELED; - } - if (Big(size).lte(totalFilled)) { + if (Big(totalFilled).gte(size)) { return OrderStatus.FILLED; + } else if (orderFlags === ORDER_FLAG_LONG_TERM) { // 1. Stateful Order + if (isCanceled) { + return OrderStatus.BEST_EFFORT_CANCELED; + } + return OrderStatus.OPEN; + } else if (timeInForce === TimeInForce.FOK) { // 2. Short-term FOK + logger.error({ + at: 'orderFillHandler#getOrderStatus', + message: 'FOK orders should never be partially filled', + blockHeight: this.block.height, + transactionIndex: this.indexerTendermintEvent.transactionIndex, + eventIndex: this.indexerTendermintEvent.eventIndex, + }); + return OrderStatus.CANCELED; + } else if (timeInForce === TimeInForce.IOC) { // 3. Short-term IOC + return OrderStatus.CANCELED; + } else if (isCanceled) { // 4. Short-term Limit & Post-only + return OrderStatus.BEST_EFFORT_CANCELED; } return OrderStatus.OPEN; } diff --git a/indexer/services/ender/src/scripts/dydx_get_order_status.sql b/indexer/services/ender/src/scripts/dydx_get_order_status.sql index 10aed2b394..354b1f5577 100644 --- a/indexer/services/ender/src/scripts/dydx_get_order_status.sql +++ b/indexer/services/ender/src/scripts/dydx_get_order_status.sql @@ -1,19 +1,39 @@ /** - Returns the order status given the total filled amount, the order size and whether the order was cancelled. + Returns the order status given the total filled amount, the order size, whether the order was + cancelled, order flags, and time in force. + * The obvious case is if totalFilled >= size, then the order status should always be `FILLED`. + * The difficult case is if totalFilled < size after a fill, then we need to keep the following + * cases in mind: + * 1. Stateful Orders - All cancelations are on-chain events, so the order can be `OPEN` or + * `BEST_EFFORT_CANCELED` if the order is in the CanceledOrdersCache. + * 2. Short-term FOK - FOK orders can never be `OPEN`, since they don't rest on the orderbook, so + * totalFilled cannot be < size. + * 3. Short-term IOC - Protocol guarantees that an IOC order will only ever be filled in a single + * block, so status should be `CANCELED`. + * 4. Short-term Limit & Post-only - If the order is in the CanceledOrdersCache, then it should be + * set to `BEST_EFFORT_CANCELED`, otherwise `OPEN`. */ -CREATE OR REPLACE FUNCTION get_order_status(total_filled numeric, size numeric, is_cancelled boolean) +CREATE OR REPLACE FUNCTION get_order_status(total_filled numeric, size numeric, is_cancelled boolean, order_flags bigint, time_in_force text) RETURNS text AS $$ -DECLARE - order_status text; BEGIN - IF is_cancelled = true THEN - order_status = 'BEST_EFFORT_CANCELED'; - ELSIF total_filled >= size THEN - order_status = 'FILLED'; + IF total_filled >= size THEN + RETURN 'FILLED'; + /** Order flag of 64 is a stateful term order */ + ELSIF order_flags = 64 THEN /** 1. Stateful Order */ + IF is_cancelled THEN + RETURN 'BEST_EFFORT_CANCELED'; + ELSE + RETURN 'OPEN'; + END IF; + ELSIF time_in_force = 'FOK' THEN /** 2. Short-term FOK */ + /** TODO(IND-439): Match knex and SQL behavior, have this log and return CANCELED */ + RAISE EXCEPTION 'FOK orders should never be partially filled'; + ELSIF time_in_force = 'IOC' THEN /** 3. Short-term IOC */ + RETURN 'CANCELED'; + ELSIF is_cancelled THEN /** 4. Short-term Limit & Postonly */ + RETURN 'BEST_EFFORT_CANCELED'; ELSE - order_status = 'OPEN'; + RETURN 'OPEN'; END IF; - - RETURN order_status; END; $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; diff --git a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql index 20d8248af7..402a068075 100644 --- a/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql +++ b/indexer/services/ender/src/scripts/dydx_order_fill_handler_per_order.sql @@ -99,7 +99,7 @@ BEGIN IF FOUND THEN order_record."totalFilled" = total_filled; - order_record."status" = get_order_status(total_filled, order_record.size, is_cancelled); + order_record."status" = get_order_status(total_filled, order_record.size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); UPDATE orders SET @@ -125,7 +125,7 @@ BEGIN order_record."type" = 'LIMIT'; /* TODO: Add additional order types once we support */ order_record."totalFilled" = fill_amount; - order_record."status" = get_order_status(fill_amount, order_size, is_cancelled); + order_record."status" = get_order_status(fill_amount, order_size, is_cancelled, order_record."orderFlags", order_record."timeInForce"); order_record."createdAtHeight" = block_height; order_record."updatedAt" = block_time; order_record."updatedAtHeight" = block_height;