From a5cb1acd9599b1870414a5c6fbb71ec2f417bcd5 Mon Sep 17 00:00:00 2001
From: GuticaStefan <stefan.gutica@gmail.com>
Date: Mon, 19 Aug 2024 12:34:14 +0300
Subject: [PATCH] Divide requests for nodes and tokens in smaller requests
 containing maximum 1000 elements and bug fixes

---
 src/endpoints/nodes/node.service.ts   | 20 +++++++++++---------
 src/endpoints/tokens/token.service.ts | 23 ++++++++++++-----------
 2 files changed, 23 insertions(+), 20 deletions(-)

diff --git a/src/endpoints/nodes/node.service.ts b/src/endpoints/nodes/node.service.ts
index 661a9dde8..3f1b6401a 100644
--- a/src/endpoints/nodes/node.service.ts
+++ b/src/endpoints/nodes/node.service.ts
@@ -25,7 +25,7 @@ import { NodeAuction } from "./entities/node.auction";
 import { NodeAuctionFilter } from "./entities/node.auction.filter";
 import { Identity } from "../identities/entities/identity";
 import { NodeSortAuction } from "./entities/node.sort.auction";
-import { ApiService, ApiSettings } from "@multiversx/sdk-nestjs-http";
+import { ApiService } from "@multiversx/sdk-nestjs-http";
 
 @Injectable()
 export class NodeService {
@@ -400,22 +400,24 @@ export class NodeService {
 
   private async getAllNodesFromApi(): Promise<Node[]> {
     try {
-      const requestNodesCount = await this.apiService.get(this.apiConfigService.getNodesFetchServiceUrl() + '/nodes/count');
+      const requestNodesCount = await this.apiService.get(`${this.apiConfigService.getNodesFetchServiceUrl()}/nodes/count`);
       const nodesCount = requestNodesCount.data;
 
-      const requestUrlParams = new ApiSettings();
-      requestUrlParams.params = {
-        size: nodesCount,
-      };
+      const nodes = [];
+      let from = 0, size = nodesCount <= 1000 ? nodesCount : 1000;
 
-      const requestNodes = await this.apiService.get(this.apiConfigService.getNodesFetchServiceUrl() + '/nodes', requestUrlParams);
-      const nodes = requestNodes.data;
+      while (size) {
+        const requestNodes = await this.apiService.get(`${this.apiConfigService.getTokensFetchServiceUrl()}/nodes`, { params: { from: from, size: size } });
+        nodes.push(...requestNodes.data);
+        from += size;
+        size = nodesCount <= from + 1000 ? nodesCount - from : 1000;
+      }
 
       return nodes;
     } catch (error) {
       this.logger.error('An unhandled error occurred when getting nodes from API');
       this.logger.error(error);
-      return [];
+      throw error;
     }
   }
 
diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts
index 14826e868..a25eda51d 100644
--- a/src/endpoints/tokens/token.service.ts
+++ b/src/endpoints/tokens/token.service.ts
@@ -20,7 +20,7 @@ import { TokenSort } from "./entities/token.sort";
 import { TokenWithRoles } from "./entities/token.with.roles";
 import { TokenWithRolesFilter } from "./entities/token.with.roles.filter";
 import { AddressUtils, BinaryUtils, NumberUtils, TokenUtils } from "@multiversx/sdk-nestjs-common";
-import { ApiService, ApiSettings, ApiUtils } from "@multiversx/sdk-nestjs-http";
+import { ApiService, ApiUtils } from "@multiversx/sdk-nestjs-http";
 import { CacheService } from "@multiversx/sdk-nestjs-cache";
 import { IndexerService } from "src/common/indexer/indexer.service";
 import { OriginLogger } from "@multiversx/sdk-nestjs-common";
@@ -720,12 +720,11 @@ export class TokenService {
   }
 
   async getAllTokensRaw(): Promise<TokenDetailed[]> {
-    this.logger.log(`Starting to fetch all tokens`);
-
     if (this.apiConfigService.isTokensFetchFeatureEnabled()) {
       return await this.getAllTokensFromApi();
     }
 
+    this.logger.log(`Starting to fetch all tokens`);
     const tokensProperties = await this.esdtService.getAllFungibleTokenProperties();
     let tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties));
 
@@ -863,22 +862,24 @@ export class TokenService {
 
   private async getAllTokensFromApi(): Promise<TokenDetailed[]> {
     try {
-      const requestTokensCount = await this.apiService.get(this.apiConfigService.getTokensFetchServiceUrl() + '/tokens/count');
+      const requestTokensCount = await this.apiService.get(`${this.apiConfigService.getTokensFetchServiceUrl()}/tokens/count`);
       const tokensCount = requestTokensCount.data;
 
-      const requestUrlParams = new ApiSettings();
-      requestUrlParams.params = {
-        size: tokensCount,
-      };
+      const tokens = [];
+      let from = 0, size = tokensCount <= 1000 ? tokensCount : 1000;
 
-      const requestTokens = await this.apiService.get(this.apiConfigService.getTokensFetchServiceUrl() + '/tokens', requestUrlParams);
-      const tokens = requestTokens.data;
+      while (size) {
+        const requestTokens = await this.apiService.get(`${this.apiConfigService.getTokensFetchServiceUrl()}/tokens`, { params: { from: from, size: size } });
+        tokens.push(...requestTokens.data);
+        from += size;
+        size = tokensCount <= from + 1000 ? tokensCount - from : 1000;
+      }
 
       return tokens;
     } catch (error) {
       this.logger.error('An unhandled error occurred when getting tokens from API');
       this.logger.error(error);
-      return [];
+      throw error;
     }
   }