Merge pull request #1319 from multiversx/SERVICES-2563-integrate-asse…

…ts-cdn integrate assets cdn
multiversx · Nov 4, 2024 · 759c51e · 759c51e
2 parents adaed8c + 5e54328
commit 759c51e
Show file tree

Hide file tree

Showing 9 changed files with 98 additions and 184 deletions.
diff --git a/config/config.devnet-old.yaml b/config/config.devnet-old.yaml
@@ -87,6 +87,7 @@ urls:
   ipfs: 'https://ipfs.io/ipfs'
   socket: 'devnet-socket-api.multiversx.com'
   maiarId: 'https://devnet-old-id-api.multiversx.com'
+  assetsCdn: 'https://tools.multiversx.com/assets-cdn'
 indexer:
   type: 'elastic'
   maxPagination: 10000

diff --git a/config/config.devnet.yaml b/config/config.devnet.yaml
@@ -49,6 +49,9 @@ features:
   dataApi:
     enabled: false
     serviceUrl: 'https://devnet-data-api.multiversx.com'
+  assetsFetch:
+    enabled: true
+    assetesUrl: 'https://tools.multiversx.com/assets-cdn'
   auth:
     enabled: false
     maxExpirySeconds: 86400

diff --git a/config/config.mainnet.yaml b/config/config.mainnet.yaml
@@ -106,6 +106,9 @@ features:
   providersFetch:
     enabled: true
     serviceUrl: 'https://api.multiversx.com'
+  assetsFetch:
+    enabled: true
+    assetesUrl: 'https://tools.multiversx.com/assets-cdn'
 image:
   width: 600
   height: 600

diff --git a/config/config.testnet.yaml b/config/config.testnet.yaml
@@ -105,6 +105,9 @@ features:
   providersFetch:
     enabled: true
     serviceUrl: 'https://testnet-api.multiversx.com'
+  assetsFetch:
+    enabled: true
+    assetesUrl: 'https://tools.multiversx.com/assets-cdn'
 image:
   width: 600
   height: 600

diff --git a/src/common/api-config/api.config.service.ts b/src/common/api-config/api.config.service.ts
@@ -866,6 +866,14 @@ export class ApiConfigService {
     return deepHistoryUrl;
   }
 
+  isAssetsCdnFeatureEnabled(): boolean {
+    return this.configService.get<boolean>('features.assetsFetch.enabled') ?? false;
+  }
+
+  getAssetsCdnUrl(): string {
+    return this.configService.get<string>('features.assetsFetch.assetesUrl') ?? 'https://tools.multiversx.com/assets-cdn';
+  }
+
   isTokensFetchFeatureEnabled(): boolean {
     return this.configService.get<boolean>('features.tokensFetch.enabled') ?? false;
   }

diff --git a/src/common/assets/assets.service.ts b/src/common/assets/assets.service.ts
@@ -1,131 +1,28 @@
 import { Injectable } from "@nestjs/common";
-import simpleGit, { SimpleGit, SimpleGitOptions } from 'simple-git';
 import { CacheInfo } from "src/utils/cache.info";
 import { TokenAssets } from "src/common/assets/entities/token.assets";
-import { ApiConfigService } from "../api-config/api.config.service";
 import { AccountAssets } from "./entities/account.assets";
 import { CacheService } from "@multiversx/sdk-nestjs-cache";
-import { FileUtils, OriginLogger } from "@multiversx/sdk-nestjs-common";
-import { ApiUtils } from "@multiversx/sdk-nestjs-http";
 import { MexPair } from "src/endpoints/mex/entities/mex.pair";
 import { Identity } from "src/endpoints/identities/entities/identity";
 import { MexFarm } from "src/endpoints/mex/entities/mex.farm";
 import { MexSettings } from "src/endpoints/mex/entities/mex.settings";
 import { DnsContracts } from "src/utils/dns.contracts";
-import { NftRankAlgorithm } from "./entities/nft.rank.algorithm";
 import { NftRank } from "./entities/nft.rank";
 import { MexStakingProxy } from "src/endpoints/mex/entities/mex.staking.proxy";
 import { Provider } from "src/endpoints/providers/entities/provider";
-
-const rimraf = require("rimraf");
-const path = require('path');
-const fs = require('fs');
+import { ApiService } from "@multiversx/sdk-nestjs-http";
+import { ApiConfigService } from "../api-config/api.config.service";
+import { KeybaseIdentity } from "../keybase/entities/keybase.identity";
 
 @Injectable()
 export class AssetsService {
-  private readonly logger = new OriginLogger(AssetsService.name);
-
   constructor(
-    private readonly cachingService: CacheService,
     private readonly apiConfigService: ApiConfigService,
+    private readonly apiService: ApiService,
+    private readonly cachingService: CacheService,
   ) { }
 
-  checkout(): Promise<void> {
-    const localGitPath = 'dist/repos/assets';
-    const logger = this.logger;
-    return new Promise((resolve, reject) => {
-      rimraf(localGitPath, function () {
-        logger.log("done deleting");
-
-        const options: Partial<SimpleGitOptions> = {
-          baseDir: process.cwd(),
-          binary: 'git',
-          maxConcurrentProcesses: 6,
-        };
-
-        // when setting all options in a single object
-        const git: SimpleGit = simpleGit(options);
-
-        // eslint-disable-next-line @typescript-eslint/no-floating-promises
-        git.outputHandler((_, stdout, stderr) => {
-          stdout.pipe(process.stdout);
-          stderr.pipe(process.stderr);
-
-          stdout.on('data', (data) => {
-            // Print data
-            logger.log(data.toString('utf8'));
-          });
-        }).clone('https://github.com/multiversx/mx-assets.git', localGitPath, undefined, (err) => {
-          if (err) {
-            reject(err);
-          } else {
-            resolve();
-          }
-        });
-      });
-    });
-  }
-
-  private readTokenAssetDetails(tokenIdentifier: string, assetPath: string): TokenAssets {
-    const infoPath = path.join(assetPath, 'info.json');
-    const info = JSON.parse(fs.readFileSync(infoPath));
-
-    return new TokenAssets({
-      ...info,
-      pngUrl: this.getImageUrl(tokenIdentifier, 'logo.png'),
-      svgUrl: this.getImageUrl(tokenIdentifier, 'logo.svg'),
-    });
-  }
-
-  private readTokenRanks(assetPath: string): NftRank[] | undefined {
-    const ranksPath = path.join(assetPath, 'ranks.json');
-    if (fs.existsSync(ranksPath)) {
-      return JSON.parse(fs.readFileSync(ranksPath));
-    }
-
-    return undefined;
-  }
-
-  private readAccountAssets(path: string): AccountAssets {
-    const jsonContents = fs.readFileSync(path);
-    const json = JSON.parse(jsonContents);
-
-    return ApiUtils.mergeObjects(new AccountAssets(), json);
-  }
-
-  private getImageUrl(tokenIdentifier: string, name: string) {
-    if (['mainnet', 'devnet', 'testnet'].includes(this.apiConfigService.getNetwork())) {
-      return `${this.apiConfigService.getExternalMediaUrl()}/tokens/asset/${tokenIdentifier}/${name}`;
-    }
-
-    return `https://raw.githubusercontent.com/multiversx/mx-assets/master/${this.apiConfigService.getNetwork()}/tokens/${tokenIdentifier}/${name}`;
-  }
-
-  private getTokenAssetsPath() {
-    return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('tokens'));
-  }
-
-  private getAccountAssetsPath() {
-    return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('accounts'));
-  }
-
-  getIdentityAssetsPath() {
-    return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('identities'));
-  }
-
-  getIdentityInfoJsonPath(identity: string): string {
-    return path.join(this.getIdentityAssetsPath(), identity, 'info.json');
-  }
-
-  private getRelativePath(name: string): string {
-    const network = this.apiConfigService.getNetwork();
-    if (network !== 'mainnet') {
-      return path.join(network, name);
-    }
-
-    return name;
-  }
-
   async getAllTokenAssets(): Promise<{ [key: string]: TokenAssets }> {
     return await this.cachingService.getOrSet(
       CacheInfo.TokenAssets.key,
@@ -134,32 +31,27 @@ export class AssetsService {
     );
   }
 
-  getAllTokenAssetsRaw(): { [key: string]: TokenAssets } {
-    const tokensPath = this.getTokenAssetsPath();
-    if (!fs.existsSync(tokensPath)) {
+  async getAllTokenAssetsRaw(): Promise<{ [key: string]: TokenAssets }> {
+    if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) {
       return {};
     }
 
-    const tokenIdentifiers = FileUtils.getDirectories(tokensPath);
+    const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl();
+    const network = this.apiConfigService.getNetwork();
+
+    const { data: assetsRaw } = await this.apiService.get(`${assetsCdnUrl}/${network}/tokens`);
 
-    // for every folder, create a TokenAssets entity with the contents of info.json and the urls from github
     const assets: { [key: string]: TokenAssets } = {};
-    for (const tokenIdentifier of tokenIdentifiers) {
-      const tokenPath = path.join(tokensPath, tokenIdentifier);
-      try {
-        assets[tokenIdentifier] = this.readTokenAssetDetails(tokenIdentifier, tokenPath);
-      } catch (error) {
-        this.logger.error(`An error occurred while reading assets for token with identifier '${tokenIdentifier}'`);
-        this.logger.error(error);
-      }
+    for (const asset of assetsRaw) {
+      const { identifier, ...details } = asset;
+      assets[identifier] = new TokenAssets(details);
     }
 
     return assets;
   }
 
   async getCollectionRanks(identifier: string): Promise<NftRank[] | undefined> {
     const allCollectionRanks = await this.getAllCollectionRanks();
-
     return allCollectionRanks[identifier];
   }
 
@@ -172,19 +64,23 @@ export class AssetsService {
   }
 
   async getAllCollectionRanksRaw(): Promise<{ [key: string]: NftRank[] }> {
-    const allTokenAssets = await this.getAllTokenAssets();
+    if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) {
+      return {};
+    }
+
+    const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl();
+    const network = this.apiConfigService.getNetwork();
+
+    const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/tokens`);
 
     const result: { [key: string]: NftRank[] } = {};
-    const assetsPath = this.getTokenAssetsPath();
-
-    for (const identifier of Object.keys(allTokenAssets)) {
-      const assets = allTokenAssets[identifier];
-      if (assets.preferredRankAlgorithm === NftRankAlgorithm.custom) {
-        const tokenAssetsPath = path.join(assetsPath, identifier);
-        const ranks = this.readTokenRanks(tokenAssetsPath);
-        if (ranks) {
-          result[identifier] = ranks;
-        }
+
+    for (const asset of assets) {
+      if (asset.ranks && asset.ranks.length > 0) {
+        result[asset.identifier] = asset.ranks.map((rank: any) => new NftRank({
+          identifier: rank.identifier,
+          rank: rank.rank,
+        }));
       }
     }
 
@@ -199,37 +95,23 @@ export class AssetsService {
     );
   }
 
-  getAllAccountAssetsRaw(providers?: Provider[], identities?: Identity[], pairs?: MexPair[], farms?: MexFarm[], mexSettings?: MexSettings, stakingProxies?: MexStakingProxy[]): { [key: string]: AccountAssets } {
-    const accountAssetsPath = this.getAccountAssetsPath();
-    if (!fs.existsSync(accountAssetsPath)) {
+  async getAllAccountAssetsRaw(providers?: Provider[], identities?: Identity[], pairs?: MexPair[], farms?: MexFarm[], mexSettings?: MexSettings, stakingProxies?: MexStakingProxy[]): Promise<{ [key: string]: AccountAssets }> {
+    if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) {
       return {};
     }
-    const fileNames = FileUtils.getFiles(accountAssetsPath);
 
-    const allAssets: { [key: string]: AccountAssets } = {};
-    for (const fileName of fileNames) {
-      if (fileName.includes(".gitkeep")) {
-        continue;
-      }
-      const assetsPath = path.join(accountAssetsPath, fileName);
-      const address = fileName.removeSuffix('.json');
-      try {
-        const assets = this.readAccountAssets(assetsPath);
-        if (assets.icon) {
-          const relativePath = this.getRelativePath(`accounts/icons/${assets.icon}`);
-          assets.iconPng = `https://raw.githubusercontent.com/multiversx/mx-assets/master/${relativePath}.png`;
-          assets.iconSvg = `https://raw.githubusercontent.com/multiversx/mx-assets/master/${relativePath}.svg`;
-
-          delete assets.icon;
-        }
+    const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl();
+    const network = this.apiConfigService.getNetwork();
 
-        allAssets[address] = assets;
-      } catch (error) {
-        this.logger.error(`An error occurred while reading assets for account with address '${address}'`);
-        this.logger.error(error);
-      }
+    const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/accounts`);
+
+    const allAssets: { [key: string]: AccountAssets } = {};
+    for (const asset of assets) {
+      const { address, ...details } = asset;
+      allAssets[address] = new AccountAssets(details);
     }
 
+    // Populate additional assets from other sources if available
     if (providers && identities) {
       for (const provider of providers) {
         const identity = identities.find(x => x.identity === provider.identity);
@@ -304,13 +186,34 @@ export class AssetsService {
   }
 
   async getTokenAssets(tokenIdentifier: string): Promise<TokenAssets | undefined> {
-    // get the dictionary from the local cache
     const assets = await this.getAllTokenAssets();
-
-    // if the tokenIdentifier key exists in the dictionary, return the associated value, else undefined
     return assets[tokenIdentifier];
   }
 
+  async getAllIdentitiesRaw(): Promise<{ [key: string]: KeybaseIdentity }> {
+    if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) {
+      return {};
+    }
+
+    const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl();
+    const network = this.apiConfigService.getNetwork();
+
+    const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/identities`);
+
+    const allAssets: { [key: string]: KeybaseIdentity } = {};
+    for (const asset of assets) {
+      const { identity, ...details } = asset;
+      allAssets[identity] = new KeybaseIdentity(details);
+    }
+
+    return allAssets;
+  }
+
+  async getIdentityInfo(identity: string): Promise<KeybaseIdentity | null> {
+    const allIdentities = await this.getAllIdentitiesRaw();
+    return allIdentities[identity] || null;
+  }
+
   createAccountAsset(name: string, tags: string[]): AccountAssets {
     return new AccountAssets({
       name: name,

diff --git a/src/common/assets/entities/nft.rank.ts b/src/common/assets/entities/nft.rank.ts
@@ -3,6 +3,12 @@ import { ApiProperty } from "@nestjs/swagger";
 
 @ObjectType("NftRank", { description: "NFT rank object type" })
 export class NftRank {
+  constructor(init?: Partial<NftRank>) {
+    if (init) {
+      Object.assign(this, init);
+    }
+  }
+
   @Field(() => String, { description: 'NFT identifier' })
   @ApiProperty({ type: String })
   identifier: string = '';