feat: suggested fees consumer (#176)

* feat: add `suggestedRelayerFeePct` column to deposit

* feat: add `SuggestedFeesConsumer`

* refactor: review requests

- move suggested fees service into nearer located file
- remove if-clause to terminate suggested relayer fee population if already exists

migrations/1670847543409-Deposit.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Deposit1670847543409 implements MigrationInterface {
+  name = "Deposit1670847543409";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "deposit" ADD "suggestedRelayerFeePct" numeric`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "deposit" DROP COLUMN "suggestedRelayerFeePct"`);
+  }
+}

src/modules/configuration/index.ts

@@ -93,6 +93,10 @@ export const configValues = () => ({
     clientSecret: process.env.DISCORD_CLIENT_SECRET,
     redirectUri: process.env.DISCORD_REDIRECT_URI,
   },
+  suggestedFees: {
+    apiUrl: process.env.SUGGESTED_FEES_API_URL || "https://across.to/api/suggested-fees",
+    fallbackThresholdHours: Number(process.env.SUGGESTED_FEES_FALLBACK_THRESHOLD_HOURS || "24"),
+  },
 });
 
 export default registerAs("config", () => configValues());

src/modules/scraper/adapter/across-serverless-api/suggested-fees-service.ts

@@ -0,0 +1,38 @@
+import { HttpService } from "@nestjs/axios";
+import { Injectable } from "@nestjs/common";
+
+import { AppConfig } from "../../../configuration/configuration.service";
+
+type SuggestedFeesApiParams = {
+  amount: string;
+  token: string;
+  destinationChainId: number;
+  originChainId: number;
+};
+
+type SuggestedFeesApiResponse = {
+  data: {
+    capitalFeePct: string;
+    capitalFeeTotal: string;
+    relayGasFeePct: string;
+    relayGasFeeTotal: string;
+    relayFeePct: string;
+    relayFeeTotal: string;
+    lpFeePct: string;
+    timestamp: string;
+    isAmountTooLow: boolean;
+  };
+};
+
+@Injectable()
+export class SuggestedFeesService {
+  constructor(private appConfig: AppConfig, private httpService: HttpService) {}
+
+  public async getFromApi(params: SuggestedFeesApiParams) {
+    const response = await this.httpService.axiosRef.get<SuggestedFeesApiParams, SuggestedFeesApiResponse>(
+      this.appConfig.values.suggestedFees.apiUrl,
+      { params },
+    );
+    return response?.data;
+  }
+}

src/modules/scraper/adapter/messaging/BlockNumberConsumer.ts

@@ -4,7 +4,13 @@ import { Job } from "bull";
 import { InjectRepository } from "@nestjs/typeorm";
 import { Repository } from "typeorm";
 import { EthProvidersService } from "../../../web3/services/EthProvidersService";
-import { BlockNumberQueueMessage, DepositReferralQueueMessage, ScraperQueue, TokenDetailsQueueMessage } from ".";
+import {
+  BlockNumberQueueMessage,
+  DepositReferralQueueMessage,
+  ScraperQueue,
+  TokenDetailsQueueMessage,
+  SuggestedFeesQueueMessage,
+} from ".";
 import { Deposit } from "../../model/deposit.entity";
 import { ScraperQueuesService } from "../../service/ScraperQueuesService";
 
@@ -31,6 +37,9 @@ export class BlockNumberConsumer {
     await this.scraperQueuesService.publishMessage<DepositReferralQueueMessage>(ScraperQueue.DepositReferral, {
       depositId: deposit.id,
     });
+    await this.scraperQueuesService.publishMessage<SuggestedFeesQueueMessage>(ScraperQueue.SuggestedFees, {
+      depositId: deposit.id,
+    });
   }
 
   @OnQueueFailed()

src/modules/scraper/adapter/messaging/SuggestedFeesConsumer.ts

@@ -0,0 +1,66 @@
+import { OnQueueFailed, Process, Processor } from "@nestjs/bull";
+import { Logger } from "@nestjs/common";
+import { Job } from "bull";
+import { InjectRepository } from "@nestjs/typeorm";
+import { Repository } from "typeorm";
+import { DateTime } from "luxon";
+import { utils } from "ethers";
+
+import { SuggestedFeesService } from "../across-serverless-api/suggested-fees-service";
+import { SuggestedFeesQueueMessage, ScraperQueue } from ".";
+import { Deposit } from "../../model/deposit.entity";
+import { AppConfig } from "../../../configuration/configuration.service";
+
+@Processor(ScraperQueue.SuggestedFees)
+export class SuggestedFeesConsumer {
+  private logger = new Logger(SuggestedFeesConsumer.name);
+
+  constructor(
+    private appConfig: AppConfig,
+    private suggestedFeesService: SuggestedFeesService,
+    @InjectRepository(Deposit) private depositRepository: Repository<Deposit>,
+  ) {}
+
+  @Process()
+  private async process(job: Job<SuggestedFeesQueueMessage>) {
+    const { depositId } = job.data;
+
+    const deposit = await this.depositRepository.findOne({
+      where: { id: depositId },
+    });
+
+    if (!deposit) {
+      this.logger.verbose(`${ScraperQueue.SuggestedFees} deposit with id '${depositId}' does not exist in db`);
+      return;
+    }
+
+    if (!deposit.depositDate) {
+      throw new Error(`Deposit with id '${depositId}' needs 'depositDate' entry in order to fetch suggested fees`);
+    }
+
+    let suggestedRelayerFeePct: string;
+
+    const diffToNowHours = DateTime.fromJSDate(deposit.depositDate).diffNow().as("hours");
+    if (Math.abs(diffToNowHours) >= this.appConfig.values.suggestedFees.fallbackThresholdHours) {
+      // Due to the inability to retrieve historic suggested fees, we fallback
+      // to 1bp for deposits that were made more than configured hours ago.
+      suggestedRelayerFeePct = utils.parseEther("0.0001").toString();
+    } else {
+      // For deposits that were made tolerable hours ago, we assume somewhat constant suggested fees.
+      const suggestedFeesFromApi = await this.suggestedFeesService.getFromApi({
+        amount: deposit.amount,
+        token: deposit.tokenAddr,
+        destinationChainId: deposit.destinationChainId,
+        originChainId: deposit.sourceChainId,
+      });
+      suggestedRelayerFeePct = suggestedFeesFromApi.relayFeePct;
+    }
+
+    await this.depositRepository.update({ id: depositId }, { suggestedRelayerFeePct });
+  }
+
+  @OnQueueFailed()
+  private onQueueFailed(job: Job, error: Error) {
+    this.logger.error(`${ScraperQueue.SuggestedFees} ${JSON.stringify(job.data)} failed: ${error}`);
+  }
+}

src/modules/scraper/adapter/messaging/index.ts

@@ -8,6 +8,7 @@ export enum ScraperQueue {
   DepositFilledDate = "DepositFilledDate",
   MerkleDistributorBlocksEvents = "MerkleDistributorBlocksEvents",
   DepositAcxPrice = "DepositAcxPrice",
+  SuggestedFees = "SuggestedFees",
 }
 
 export type BlocksEventsQueueMessage = {
@@ -55,3 +56,7 @@ export type DepositFilledDateQueueMessage = {
 export type DepositAcxPriceQueueMessage = {
   depositId: number;
 };
+
+export type SuggestedFeesQueueMessage = {
+  depositId: number;
+};

src/modules/scraper/entry-point/http/controller.ts

@@ -13,6 +13,7 @@ import {
   TokenPriceQueueMessage,
   TokenDetailsQueueMessage,
   DepositAcxPriceQueueMessage,
+  SuggestedFeesQueueMessage,
 } from "../../adapter/messaging";
 import { ScraperService } from "../../service";
 import { ScraperQueuesService } from "../../service/ScraperQueuesService";
@@ -23,6 +24,7 @@ import {
   SubmitDepositFilledDateBody,
   ProcessBlockNumberBody,
   SubmitDepositAcxPriceBody,
+  SubmitSuggestedFeesBody,
 } from "./dto";
 
 @Controller()
@@ -146,4 +148,19 @@ export class ScraperController {
       });
     }
   }
+
+  @Post("scraper/suggested-fees")
+  @ApiTags("scraper")
+  @Roles(Role.Admin)
+  @UseGuards(JwtAuthGuard, RolesGuard)
+  @ApiBearerAuth()
+  async submitSuggestedFeesJob(@Body() body: SubmitSuggestedFeesBody) {
+    const { fromDepositId, toDepositId } = body;
+
+    for (let depositId = fromDepositId; depositId <= toDepositId; depositId++) {
+      await this.scraperQueuesService.publishMessage<SuggestedFeesQueueMessage>(ScraperQueue.SuggestedFees, {
+        depositId,
+      });
+    }
+  }
 }

src/modules/scraper/entry-point/http/dto.ts

@@ -64,3 +64,13 @@ export class ProcessBlockNumberBody {
   @ApiProperty({ example: 2 })
   toDepositId: number;
 }
+
+export class SubmitSuggestedFeesBody {
+  @IsInt()
+  @ApiProperty({ example: 1 })
+  fromDepositId: number;
+
+  @IsInt()
+  @ApiProperty({ example: 2 })
+  toDepositId: number;
+}

src/modules/scraper/model/deposit.entity.ts

@@ -62,6 +62,9 @@ export class Deposit {
   @Column({ type: "decimal" })
   depositRelayerFeePct?: string;
 
+  @Column({ type: "decimal", nullable: true })
+  suggestedRelayerFeePct?: string;
+
   @Column({ type: "decimal", default: 0 })
   realizedLpFeePctCapped: string;
 

src/modules/scraper/module.ts

@@ -27,11 +27,14 @@ import { MerkleDistributorProcessedBlock } from "./model/MerkleDistributorProces
 import { ScraperService } from "./service";
 import { ScraperQueuesService } from "./service/ScraperQueuesService";
 import { DepositAcxPriceConsumer } from "./adapter/messaging/DepositAcxPriceConsumer";
+import { SuggestedFeesConsumer } from "./adapter/messaging/SuggestedFeesConsumer";
+import { SuggestedFeesService } from "./adapter/across-serverless-api/suggested-fees-service";
 
 @Module({
   providers: [
     ScraperService,
     ScraperQueuesService,
+    SuggestedFeesService,
     BlocksEventsConsumer,
     MerkleDistributorBlocksEventsConsumer,
     FillEventsConsumer,
@@ -41,6 +44,7 @@ import { DepositAcxPriceConsumer } from "./adapter/messaging/DepositAcxPriceCons
     TokenPriceConsumer,
     DepositFilledDateConsumer,
     DepositAcxPriceConsumer,
+    SuggestedFeesConsumer,
     DepositFixture,
     ClaimFixture,
   ],
@@ -84,6 +88,9 @@ import { DepositAcxPriceConsumer } from "./adapter/messaging/DepositAcxPriceCons
     BullModule.registerQueue({
       name: ScraperQueue.DepositFilledDate,
     }),
+    BullModule.registerQueue({
+      name: ScraperQueue.SuggestedFees,
+    }),
   ],
   exports: [ScraperQueuesService],
   controllers: [ScraperController],

src/modules/scraper/service/ScraperQueuesService.ts

@@ -17,6 +17,7 @@ export class ScraperQueuesService {
     @InjectQueue(ScraperQueue.TokenPrice) private tokenPriceQueue: Queue,
     @InjectQueue(ScraperQueue.DepositFilledDate) private depositFilledDateQueue: Queue,
     @InjectQueue(ScraperQueue.DepositAcxPrice) private depositAcxPriceQueue: Queue,
+    @InjectQueue(ScraperQueue.SuggestedFees) private suggestedFeesQueue: Queue,
   ) {
     setInterval(() => {
       this.blocksEventsQueue
@@ -46,6 +47,9 @@ export class ScraperQueuesService {
       this.depositAcxPriceQueue
         .getJobCounts()
         .then((data) => this.logger.log(`${ScraperQueue.DepositAcxPrice} ${JSON.stringify(data)}`));
+      this.suggestedFeesQueue
+        .getJobCounts()
+        .then((data) => this.logger.log(`${ScraperQueue.SuggestedFees} ${JSON.stringify(data)}`));
     }, 1000 * 60);
   }
 
@@ -68,6 +72,8 @@ export class ScraperQueuesService {
       await this.merkleDistributorBlocksEventsQueue.add(message);
     } else if (queue === ScraperQueue.DepositAcxPrice) {
       await this.depositAcxPriceQueue.add(message);
+    } else if (queue === ScraperQueue.SuggestedFees) {
+      await this.suggestedFeesQueue.add(message);
     }
   }
 
@@ -90,6 +96,8 @@ export class ScraperQueuesService {
       await this.merkleDistributorBlocksEventsQueue.addBulk(messages.map((m) => ({ data: m })));
     } else if (queue === ScraperQueue.DepositAcxPrice) {
       await this.depositAcxPriceQueue.addBulk(messages.map((m) => ({ data: m })));
+    } else if (queue === ScraperQueue.SuggestedFees) {
+      await this.suggestedFeesQueue.addBulk(messages.map((m) => ({ data: m })));
     }
   }
 }