Merge branch 'development' into SERVICES-2694-remove-support-for-grap…

…hql-service

src/common/indexer/elastic/elastic.indexer.service.ts

@@ -404,7 +404,7 @@ export class ElasticIndexerService implements IndexerInterface {
     return await this.elasticService.getList('operations', 'hash', elasticQuery);
   }
 
-  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<any[]> {
+  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<any[]> {
     let elasticQuery = this.indexerHelper.buildAccountFilterQuery(filter);
     const sortOrder: ElasticSortOrder = !filter.order || filter.order === SortOrder.desc ? ElasticSortOrder.descending : ElasticSortOrder.ascending;
     const sort: AccountSort = filter.sort ?? AccountSort.balance;
@@ -429,6 +429,10 @@ export class ElasticIndexerService implements IndexerInterface {
 
     elasticQuery = elasticQuery.withPagination(queryPagination);
 
+    if (fields && fields.length > 0) {
+      elasticQuery = elasticQuery.withFields(fields);
+    }
+
     return await this.elasticService.getList('accounts', 'address', elasticQuery);
   }
 
@@ -805,8 +809,6 @@ export class ElasticIndexerService implements IndexerInterface {
       }
     }
 
-    console.log({ subType: filter.subType });
-
     const elasticQuery = ElasticQuery.create()
       .withMustExistCondition('identifier')
       .withMustMatchCondition('address', address)
@@ -888,7 +890,7 @@ export class ElasticIndexerService implements IndexerInterface {
   async getAllFungibleTokens(): Promise<any[]> {
     const query = ElasticQuery.create()
       .withMustMatchCondition('type', TokenType.FungibleESDT)
-      .withFields(["name", "type", "currentOwner", "numDecimals", "properties", "timestamp"])
+      .withFields(["name", "type", "currentOwner", "numDecimals", "properties", "timestamp", "ownersHistory"])
       .withMustNotExistCondition('identifier')
       .withPagination({ from: 0, size: 1000 });
 
@@ -978,6 +980,16 @@ export class ElasticIndexerService implements IndexerInterface {
     return await this.elasticService.getCount('scdeploys', elasticQuery);
   }
 
+  async getAddressesWithTransfersLast24h(): Promise<string[]> {
+    const elasticQuery = ElasticQuery.create()
+      .withFields(['address'])
+      .withPagination({ from: 0, size: 10000 })
+      .withMustExistCondition('api_transfersLast24h');
+
+    const result = await this.elasticService.getList('accounts', 'address', elasticQuery);
+    return result.map(x => x.address);
+  }
+
   async getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]> {
     const elasticQuery = this.indexerHelper.buildEventsFilter(filter)
       .withPagination(pagination)

src/common/indexer/indexer.interface.ts

@@ -104,7 +104,7 @@ export interface IndexerInterface {
 
   getAccount(address: string): Promise<Account>
 
-  getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<Account[]>
+  getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<Account[]>
 
   getAccountDeploys(pagination: QueryPagination, address: string): Promise<ScDeploy[]>
 
@@ -188,6 +188,8 @@ export interface IndexerInterface {
 
   getApplicationCount(filter: ApplicationFilter): Promise<number>
 
+  getAddressesWithTransfersLast24h(): Promise<string[]>
+
   getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]>
 
   getEvent(txHash: string): Promise<Events>

src/common/indexer/indexer.service.ts

@@ -227,8 +227,8 @@ export class IndexerService implements IndexerInterface {
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
-  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<Account[]> {
-    return await this.indexerInterface.getAccounts(queryPagination, filter);
+  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<Account[]> {
+    return await this.indexerInterface.getAccounts(queryPagination, filter, fields);
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
@@ -452,6 +452,10 @@ export class IndexerService implements IndexerInterface {
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
+  async getAddressesWithTransfersLast24h(): Promise<string[]> {
+    return await this.indexerInterface.getAddressesWithTransfersLast24h();
+  }
+
   async getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]> {
     return await this.indexerInterface.getEvents(pagination, filter);
   }

src/crons/cache.warmer/cache.warmer.service.ts

@@ -33,6 +33,7 @@ import { PoolService } from "src/endpoints/pool/pool.service";
 import * as JsonDiff from "json-diff";
 import { QueryPagination } from "src/common/entities/query.pagination";
 import { StakeService } from "src/endpoints/stake/stake.service";
+import { ApplicationMostUsed } from "src/endpoints/accounts/entities/application.most.used";
 
 @Injectable()
 export class CacheWarmerService {
@@ -377,21 +378,28 @@ export class CacheWarmerService {
   async handleUpdateAccountTransfersLast24h() {
     const batchSize = 100;
     const mostUsed = await this.accountService.getApplicationMostUsedRaw();
+    const mostUsedIndexedAccounts = await this.indexerService.getAddressesWithTransfersLast24h();
 
-    const batches = BatchUtils.splitArrayIntoChunks(mostUsed, batchSize);
+    const allAddressesToUpdate = [...mostUsed.map(item => item.address), ...mostUsedIndexedAccounts].distinct();
+    const mostUsedDictionary = mostUsed.toRecord<ApplicationMostUsed>(item => item.address);
+
+    const batches = BatchUtils.splitArrayIntoChunks(allAddressesToUpdate, batchSize);
     for (const batch of batches) {
       const accounts = await this.indexerService.getAccounts(
         new QueryPagination({ from: 0, size: batchSize }),
-        new AccountQueryOptions({ addresses: batch.map(item => item.address) }),
+        new AccountQueryOptions({ addresses: batch }),
+        ['address', 'api_transfersLast24h'],
       );
 
-      const accountsDictionary = accounts.toRecord<Account>(account => account.address);
+      const accountsDictionary = accounts.toRecord<Pick<Account, 'address' | 'api_transfersLast24h'>>(account => account.address);
+
+      for (const address of batch) {
+        const account = accountsDictionary[address];
+        const newTransfersLast24h = mostUsedDictionary[address]?.transfers24H ?? 0;
 
-      for (const item of batch) {
-        const account = accountsDictionary[item.address];
-        if (account && account.api_transfersLast24h !== item.transfers24H) {
-          this.logger.log(`Setting transferLast24h to ${item.transfers24H} for account with address '${item.address}'`);
-          await this.indexerService.setAccountTransfersLast24h(item.address, item.transfers24H);
+        if (account && account.api_transfersLast24h !== newTransfersLast24h) {
+          this.logger.log(`Setting transferLast24h to ${newTransfersLast24h} for account with address '${address}'`);
+          await this.indexerService.setAccountTransfersLast24h(address, newTransfersLast24h);
         }
       }
     }

src/endpoints/esdt/esdt.address.service.ts

@@ -306,10 +306,7 @@ export class EsdtAddressService {
         }
       }
 
-      if ([NftType.SemiFungibleESDT, NftType.MetaESDT].includes(nft.type)) {
-        nft.balance = dataSourceNft.balance;
-      }
-
+      nft.balance = dataSourceNft.balance;
       nftAccounts.push(nft);
     }
 

src/endpoints/esdt/esdt.service.ts

@@ -215,6 +215,7 @@ export class EsdtService {
       type: elasticProperties.type as EsdtType,
       subType: elasticProperties.type as EsdtSubType,
       owner: elasticProperties.currentOwner,
+      ownersHistory: elasticProperties.ownersHistory,
       decimals: elasticProperties.numDecimals,
       canUpgrade: elasticProperties.properties?.canUpgrade ?? false,
       canMint: elasticProperties.properties?.canMint ?? false,

src/endpoints/tokens/entities/token.owner.history.ts

@@ -0,0 +1,8 @@
+export class TokenOwnersHistory {
+  constructor(init?: Partial<TokenOwnersHistory>) {
+    Object.assign(this, init);
+  }
+
+  address: string = '';
+  timestamp: number = 0;
+}

src/endpoints/tokens/entities/token.properties.ts

@@ -1,6 +1,7 @@
 import { ApiProperty } from "@nestjs/swagger";
 import { EsdtType } from "../../esdt/entities/esdt.type";
 import { EsdtSubType } from "src/endpoints/esdt/entities/esdt.sub.type";
+import { TokenOwnersHistory } from "./token.owner.history";
 
 export class TokenProperties {
   constructor(init?: Partial<TokenProperties>) {
@@ -75,4 +76,7 @@ export class TokenProperties {
 
   @ApiProperty()
   timestamp: number = 0;
+
+  @ApiProperty()
+  ownersHistory: TokenOwnersHistory[] = [];
 }

src/endpoints/tokens/entities/token.ts

@@ -3,6 +3,7 @@ import { ApiProperty } from "@nestjs/swagger";
 import { TokenType } from "src/common/indexer/entities";
 import { TokenAssets } from "../../../common/assets/entities/token.assets";
 import { MexPairType } from "src/endpoints/mex/entities/mex.pair.type";
+import { TokenOwnersHistory } from "./token.owner.history";
 
 export class Token {
   constructor(init?: Partial<Token>) {
@@ -125,4 +126,8 @@ export class Token {
 
   @ApiProperty({ type: Number, nullable: true })
   tradesCount: number | undefined = undefined;
+
+  @Field(() => TokenOwnersHistory, { description: 'Token owners history.', nullable: true })
+  @ApiProperty({ type: TokenOwnersHistory, nullable: true })
+  ownersHistory: TokenOwnersHistory[] = [];
 }

src/test/unit/services/tokens.spec.ts

@@ -854,6 +854,12 @@ describe('Token Service', () => {
       canTransferNFTCreateRole: false,
       NFTCreateStopped: false,
       timestamp: 1643824710,
+      ownersHistory: [
+        {
+          address: 'erd1qqqqqqqqqqqqqpgq0lzzvt2faev4upyf586tg38s84d7zsaj2jpsglugga',
+          timestamp: 1643824710,
+        },
+      ],
     };
 
     it('should returns undefined if getTokenProperties returns undefined', async () => {
@@ -903,6 +909,12 @@ describe('Token Service', () => {
       canTransferNFTCreateRole: false,
       NFTCreateStopped: false,
       timestamp: 1643824710,
+      ownersHistory: [
+        {
+          address: 'erd1qqqqqqqqqqqqqpgq0lzzvt2faev4upyf586tg38s84d7zsaj2jpsglugga',
+          timestamp: 1643824710,
+        },
+      ],
     };
 
     const assets = {