feat: release 2022-11-23 (#153)

.env.sample

@@ -12,6 +12,7 @@ WEB3_NODE_URL_10=https://optimism-mainnet.infura.io/v3/
 WEB3_NODE_URL_288=https://boba-mainnet.gateway.pokt.network/v1/lb/
 WEB3_NODE_URL_42161=https://arbitrum-mainnet.infura.io/v3/
 WEB3_NODE_URL_137=https://polygon-mainnet.infura.io/v3/
+WEB3_NODE_URL_5=https://goerli.infura.io/v3/
 REFERRAL_DELIMITER_START_TIMESTAMP=1657290720
 ENABLE_SPOKE_POOLS_EVENTS_PROCESSING=false
 ENABLE_REFERRALS_MATERIALIZED_VIEW_REFRESH=false
@@ -27,3 +28,8 @@ DISCORD_CLIENT_ID=clientId
 DISCORD_CLIENT_SECRET=clientSecret
 # the url accessed after the Discord authorization processed is fulfilled
 DISCORD_REDIRECT_URI=http://localhost
+
+# MerkleDistributor overrides
+MERKLE_DISTRIBUTOR_CHAIN_ID=
+MERKLE_DISTRIBUTOR_ADDRESS=
+ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING=false

migrations/1667312452969-MerkleDistributorRecipient.ts

@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MerkleDistributorRecipient1667312452969 implements MigrationInterface {
+  name = "MerkleDistributorRecipient1667312452969";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE "merkle_distributor_recipient"
+        DROP CONSTRAINT "UK_merkle_distributor_recipient_merkleDistributorWindowId_addre"
+    `);
+    await queryRunner.query(`
+      ALTER TABLE "merkle_distributor_recipient"
+        ADD CONSTRAINT "UK_merkle_distributor_recipient_windowId_address"
+          UNIQUE ("merkleDistributorWindowId", "address")
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE "merkle_distributor_recipient"
+        DROP CONSTRAINT "UK_merkle_distributor_recipient_windowId_address"
+    `);
+    await queryRunner.query(`
+      ALTER TABLE "merkle_distributor_recipient"
+        ADD CONSTRAINT "UK_merkle_distributor_recipient_merkleDistributorWindowId_addre"
+          UNIQUE ("merkleDistributorWindowId", "address")
+    `);
+  }
+}

migrations/1667813310964-MerkleDistributorProcessedBlock.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MerkleDistributorProcessedBlock1667813310964 implements MigrationInterface {
+  name = "MerkleDistributorProcessedBlock1667813310964";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      CREATE TABLE "merkle_distributor_processed_block" (
+        "id" SERIAL NOT NULL,
+        "chainId" integer NOT NULL,
+        "latestBlock" integer NOT NULL,
+        "createdAt" TIMESTAMP NOT NULL DEFAULT now(),
+        CONSTRAINT "PK_fb2eb512abaadb453e1cfef109e" PRIMARY KEY ("id"))
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "merkle_distributor_processed_block"`);
+  }
+}

migrations/1667813310965-Claim.ts

@@ -0,0 +1,44 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Claim1667813310965 implements MigrationInterface {
+  name = "Claim1667813310965";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      CREATE TABLE "claim" (
+        "id" SERIAL NOT NULL,
+        "caller" character varying NOT NULL,
+        "accountIndex" integer NOT NULL,
+        "windowIndex" integer NOT NULL,
+        "account" character varying NOT NULL,
+        "rewardToken" character varying NOT NULL,
+        "blockNumber" integer NOT NULL,
+        "claimedAt" TIMESTAMP NOT NULL,
+        "createdAt" TIMESTAMP NOT NULL DEFAULT now(),
+        "updatedAt" TIMESTAMP NOT NULL DEFAULT now(),
+        "merkleDistributorWindowId" integer,
+        CONSTRAINT "UK_claim_windowIndex_accountIndex" UNIQUE (
+          "windowIndex",
+          "accountIndex"
+        ),
+        CONSTRAINT "PK_466b305cc2e591047fa1ce58f81" PRIMARY KEY ("id"))
+    `);
+    await queryRunner.query(`CREATE INDEX "IX_claim_account" ON "claim" ("account")`);
+    await queryRunner.query(`
+      ALTER TABLE "claim"
+        ADD CONSTRAINT "FK_169ca2a2e031f01f62d81dbf1a0"
+          FOREIGN KEY ("merkleDistributorWindowId")
+          REFERENCES "merkle_distributor_window"("id")
+          ON DELETE NO ACTION ON UPDATE NO ACTION
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE "claim" DROP CONSTRAINT "FK_169ca2a2e031f01f62d81dbf1a0"
+    `);
+    await queryRunner.query(`DROP TABLE "merkle_distributor_processed_block"`);
+    await queryRunner.query(`DROP INDEX "public"."IX_claim_account"`);
+    await queryRunner.query(`DROP TABLE "claim"`);
+  }
+}

migrations/1667822226389-Deposit.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Deposit1667822226389 implements MigrationInterface {
+  name = "Deposit1667822226389";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "deposit" ADD "rewardsWindowIndex" integer`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "deposit" DROP COLUMN "rewardsWindowIndex"`);
+  }
+}

migrations/1668007817974-DepositsMaterializedView.ts

@@ -0,0 +1,39 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class DepositsMaterializedView1668007817974 implements MigrationInterface {
+  name = "DepositsMaterializedView1668007817974";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP MATERIALIZED VIEW IF EXISTS "deposits_mv"`);
+    await queryRunner.query(`DROP VIEW IF EXISTS "deposit_referral_stats"`);
+    await queryRunner.query(`DROP VIEW IF EXISTS "deposits_filtered_referrals"`);
+    await queryRunner.query(`CREATE VIEW "deposits_filtered_referrals" AS 
+      SELECT 
+        d.id,
+        d."stickyReferralAddress",
+        d."depositDate",
+        d."priceId",
+        d."tokenId",
+        d.amount,
+        d."depositorAddr",
+        case when d."rewardsWindowIndex" = c."windowIndex" then d."rewardsWindowIndex" else -1 end as "claimedWindowIndex",
+        hmp."usd",
+        t.decimals
+      FROM deposit d
+      JOIN historic_market_price hmp on d."priceId" = hmp.id
+      JOIN token t on d."tokenId" = t.id
+      LEFT JOIN claim c on d."rewardsWindowIndex" = c."windowIndex" and d."referralAddress" = c."account"
+      WHERE "stickyReferralAddress" is not null
+        AND "depositDate" is not null
+        AND "tokenId" is not null
+        AND "priceId" is not null
+        AND status = 'filled';
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP MATERIALIZED VIEW "deposits_mv"`);
+    await queryRunner.query(`DROP VIEW "deposit_referral_stats"`);
+    await queryRunner.query(`DROP VIEW "deposits_filtered_referrals"`);
+  }
+}

migrations/1668339782207-DepositReferralStat.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class DepositReferralStat1668339782207 implements MigrationInterface {
+  name = "DepositReferralStat1668339782207";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "deposit_referral_stat" (
+        "id" SERIAL NOT NULL, 
+        "depositId" integer NOT NULL, 
+        "referralCount" integer NOT NULL, 
+        "referralVolume" numeric NOT NULL, 
+        "createdAt" TIMESTAMP NOT NULL DEFAULT now(), 
+        "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), CONSTRAINT "PK_74ce2ecc7d76a2e59039e94fcfc" PRIMARY KEY ("id")
+      )`,
+    );
+    await queryRunner.query(`CREATE UNIQUE INDEX "UK_drs_depositId" ON deposit_referral_stat ("depositId");`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP TABLE "deposit_referral_stat"`);
+  }
+}

migrations/1668339782208-DepositsMaterializedView.ts

@@ -0,0 +1,54 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class DepositsMaterializedView1668339782208 implements MigrationInterface {
+  name = "DepositsMaterializedView1668339782208";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP MATERIALIZED VIEW IF EXISTS "deposits_mv"`);
+    await queryRunner.query(`CREATE MATERIALIZED VIEW "deposits_mv" AS 
+      SELECT
+        d.id,
+        d."depositId",
+        d."depositTxHash",
+        d."sourceChainId",
+        d."destinationChainId",
+        d.amount,
+        t.symbol,
+        t.decimals,
+        d."depositorAddr",
+        d."rewardsWindowIndex",
+        case when d."rewardsWindowIndex" = c."windowIndex" and d."depositorAddr" = c.account then d."rewardsWindowIndex" else -1 end as "depositorClaimedWindowIndex",
+        case when d."rewardsWindowIndex" = c."windowIndex" and d."stickyReferralAddress" = c.account then d."rewardsWindowIndex" else -1 end as "referralClaimedWindowIndex",
+        d."stickyReferralAddress" AS "referralAddress",
+        d."depositDate",
+        hmp.usd AS "tokenUsdPrice",
+        (d."realizedLpFeePctCapped" / power(10, 18)) * (d.amount / power(10, t.decimals)) * hmp.usd AS "realizedLpFeeUsd",
+        (d."bridgeFeePct" / power(10, 18)) * (d.amount / power(10, t.decimals)) * hmp.usd AS "bridgeFeeUsd",
+      CASE
+        WHEN d1."referralCount" >= 20 OR d1."referralVolume" >= 500000 THEN 0.8
+        WHEN d1."referralCount" >= 10 OR d1."referralVolume" >= 250000 THEN 0.7
+        WHEN d1."referralCount" >= 5 OR d1."referralVolume" >= 100000 THEN 0.6
+        WHEN d1."referralCount" >= 3 OR d1."referralVolume" >= 50000 THEN 0.5
+        ELSE 0.4
+      END AS "referralRate",
+      CASE
+          WHEN d."depositDate" < '2022-07-22 17:00:00' THEN 3
+          ELSE 2
+      END AS multiplier
+      FROM deposit d
+      JOIN "deposit_referral_stat" d1 ON d."id" = d1."depositId"
+      JOIN token t ON d."tokenId" = t.id
+      JOIN historic_market_price hmp ON d."priceId" = hmp.id
+      LEFT JOIN claim c on d."rewardsWindowIndex" = c."windowIndex" and (d."stickyReferralAddress" = c."account" or c.account = d."depositorAddr")
+      ORDER BY d."depositDate" desc;
+    `);
+
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "UK_deposits_mv_depositId_sourceChainId" ON deposits_mv ("depositId", "sourceChainId");`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP MATERIALIZED VIEW "deposits_mv"`);
+  }
+}

package.json

@@ -26,7 +26,7 @@
     "db:migration:run": "yarn typeorm -- migration:run  -d ormconfig.ts"
   },
   "dependencies": {
-    "@across-protocol/contracts-v2": "^1.0.0",
+    "@across-protocol/contracts-v2": "^1.0.7",
     "@nestjs/axios": "^0.0.8",
     "@nestjs/bull": "^0.5.5",
     "@nestjs/cli": "^8.2.4",
@@ -48,6 +48,7 @@
     "async": "^3.2.4",
     "async-retry": "^1.3.3",
     "bignumber.js": "^9.0.2",
+    "bluebird": "^3.7.2",
     "bull": "^4.8.4",
     "cache-manager": "^4.0.0",
     "class-transformer": "^0.5.1",
@@ -76,6 +77,7 @@
   "devDependencies": {
     "@types/async": "^3.2.13",
     "@types/bignumber.js": "^5.0.0",
+    "@types/bluebird": "^3.5.37",
     "@types/cron": "^2.0.0",
     "@types/express": "^4.17.13",
     "@types/jest": "^27.4.1",

src/modules/airdrop/entry-points/http/controller.ts

@@ -91,7 +91,7 @@ export class AirdropController {
   @UseGuards(JwtAuthGuard, RolesGuard)
   @UseInterceptors(
     FileInterceptor("file", {
-      limits: { fileSize: 50 * 1024 * 1024 },
+      limits: { fileSize: 100 * 1024 * 1024 },
       dest: "./uploads",
     }),
   )

src/modules/airdrop/model/merkle-distributor-recipient.entity.ts

@@ -13,7 +13,7 @@ export type MerkleDistributorRecipientPayload = {
 
 @Entity()
 // A recipient address can't appear twice for the same window
-@Unique("UK_merkle_distributor_recipient_merkleDistributorWindowId_address", ["merkleDistributorWindowId", "address"])
+@Unique("UK_merkle_distributor_recipient_windowId_address", ["merkleDistributorWindowId", "address"])
 export class MerkleDistributorRecipient {
   @PrimaryGeneratedColumn()
   id: number;

src/modules/airdrop/model/merkle-distributor-window.entity.ts

@@ -1,5 +1,6 @@
 import { Column, CreateDateColumn, Entity, OneToMany, PrimaryGeneratedColumn, Unique } from "typeorm";
 import { MerkleDistributorRecipient } from "./merkle-distributor-recipient.entity";
+import { Claim } from "../../scraper/model/claim.entity";
 
 @Entity()
 // Don't allow duplicates of the window index
@@ -29,6 +30,9 @@ export class MerkleDistributorWindow {
   @OneToMany(() => MerkleDistributorRecipient, (recipient) => recipient.merkleDistributorWindow)
   recipients: MerkleDistributorRecipient;
 
+  @OneToMany(() => Claim, (claim) => claim.merkleDistributorWindow)
+  claims: Claim;
+
   @CreateDateColumn()
   createdAt: Date;
 }

src/modules/airdrop/services/airdrop-service.ts

@@ -345,7 +345,7 @@ export class AirdropService {
     }
 
     const wei = new BigNumber(10).pow(18);
-    communityRewards.amount = new BigNumber(amount).multipliedBy(wei).toString();
+    communityRewards.amount = new BigNumber(amount).multipliedBy(wei).toFixed();
     communityRewards.discordId = discordId;
     communityRewards.processed = true;
 
@@ -370,9 +370,9 @@ export class AirdropService {
     }
 
     const wei = new BigNumber(10).pow(18);
-    walletRewards.earlyUserRewards = new BigNumber(earlyUserRewards).multipliedBy(wei).toString();
-    walletRewards.liquidityProviderRewards = new BigNumber(liquidityProviderRewards).multipliedBy(wei).toString();
-    walletRewards.welcomeTravellerRewards = new BigNumber(welcomeTravellerRewards).multipliedBy(wei).toString();
+    walletRewards.earlyUserRewards = new BigNumber(earlyUserRewards).multipliedBy(wei).toFixed();
+    walletRewards.liquidityProviderRewards = new BigNumber(liquidityProviderRewards).multipliedBy(wei).toFixed();
+    walletRewards.welcomeTravellerRewards = new BigNumber(welcomeTravellerRewards).multipliedBy(wei).toFixed();
     walletRewards.walletAddress = walletAddress;
     walletRewards.processed = true;
 

src/modules/configuration/index.ts

@@ -39,7 +39,7 @@ export const configValues = () => ({
       // 69: process.env.WEB3_NODE_URL_69,
       // 4: process.env.WEB3_NODE_URL_4,
       // 80001: process.env.WEB3_NODE_URL_80001,
-      // 5: process.env.WEB3_NODE_URL_5,
+      5: process.env.WEB3_NODE_URL_5,
     },
     spokePoolContracts: {
       [ChainIds.mainnet]: {
@@ -63,9 +63,19 @@ export const configValues = () => ({
         startBlockNumber: 28604263,
       },
     },
+    merkleDistributor: {
+      address: process.env.MERKLE_DISTRIBUTOR_ADDRESS || "0xF633b72A4C2Fb73b77A379bf72864A825aD35b6D", // TODO: replace with mainnet
+      chainId: Number(process.env.MERKLE_DISTRIBUTOR_CHAIN_ID || "5"),
+      referralsStartWindowIndex: Number(process.env.REFERRALS_START_WINDOW_INDEX || "1"),
+      startBlockNumber: Number(process.env.MERKLE_DISTRIBUTOR_START_BLOCK || 7884371),
+    },
+    acx: {
+      address: process.env.ACX_ADDRESS || "0x40153DdFAd90C49dbE3F5c9F96f2a5B25ec67461", // TODO: replace with mainnet,
+    },
   },
   acxUsdPrice: 0.1,
   enableSpokePoolsEventsProcessing: process.env.ENABLE_SPOKE_POOLS_EVENTS_PROCESSING === "true",
+  enableMerkleDistributorEventsProcessing: process.env.ENABLE_MERKLE_DISTRIBUTOR_EVENTS_PROCESSING === "true",
   enableReferralsMaterializedViewRefresh: process.env.ENABLE_REFERRALS_MATERIALIZED_VIEW_REFRESH === "true",
   allowWalletRewardsEdit: process.env.ALLOW_WALLET_REWARDS_EDIT === "true",
   stickyReferralAddressesMechanism: process.env.STICKY_REFERRAL_ADDRESSES_MECHANISM

src/modules/database/database.providers.ts

@@ -2,6 +2,8 @@ import { TypeOrmOptionsFactory, TypeOrmModuleOptions } from "@nestjs/typeorm";
 import { Injectable } from "@nestjs/common";
 import { AppConfig } from "../configuration/configuration.service";
 import { ProcessedBlock } from "../scraper/model/ProcessedBlock.entity";
+import { MerkleDistributorProcessedBlock } from "../scraper/model/MerkleDistributorProcessedBlock.entity";
+import { Claim } from "../scraper/model/claim.entity";
 import { Block } from "../web3/model/block.entity";
 import { Deposit } from "../scraper/model/deposit.entity";
 import { Token } from "../web3/model/token.entity";
@@ -13,10 +15,16 @@ import { CommunityRewards } from "../airdrop/model/community-rewards.entity";
 import { UserWallet } from "../user/model/user-wallet.entity";
 import { MerkleDistributorRecipient } from "../airdrop/model/merkle-distributor-recipient.entity";
 import { MerkleDistributorWindow } from "../airdrop/model/merkle-distributor-window.entity";
+import { DepositReferralStats } from "../referral/model/DepositReferralStats.entity";
+import { DepositsMv } from "../deposit/model/DepositsMv.entity";
+import { DepositsFilteredReferrals } from "../referral/model/DepositsFilteredReferrals.entity";
+import { DepositReferralStat } from "../deposit/model/deposit-referral-stat.entity";
 
 // TODO: Add db entities here
 const entities = [
   ProcessedBlock,
+  MerkleDistributorProcessedBlock,
+  Claim,
   Block,
   Deposit,
   Token,
@@ -28,6 +36,10 @@ const entities = [
   UserWallet,
   MerkleDistributorWindow,
   MerkleDistributorRecipient,
+  DepositReferralStats,
+  DepositsMv,
+  DepositsFilteredReferrals,
+  DepositReferralStat,
 ];
 
 @Injectable()