diff --git a/config/config.devnet-old.yaml b/config/config.devnet-old.yaml index 0acda1a78..b066116c1 100644 --- a/config/config.devnet-old.yaml +++ b/config/config.devnet-old.yaml @@ -87,6 +87,7 @@ urls: ipfs: 'https://ipfs.io/ipfs' socket: 'devnet-socket-api.multiversx.com' maiarId: 'https://devnet-old-id-api.multiversx.com' + assetsCdn: 'https://tools.multiversx.com/assets-cdn' indexer: type: 'elastic' maxPagination: 10000 diff --git a/config/config.devnet.yaml b/config/config.devnet.yaml index c0ce425ba..48ee28c9e 100644 --- a/config/config.devnet.yaml +++ b/config/config.devnet.yaml @@ -49,6 +49,9 @@ features: dataApi: enabled: false serviceUrl: 'https://devnet-data-api.multiversx.com' + assetsFetch: + enabled: true + assetesUrl: 'https://tools.multiversx.com/assets-cdn' auth: enabled: false maxExpirySeconds: 86400 diff --git a/config/config.mainnet.yaml b/config/config.mainnet.yaml index 7bb286ea8..0cd7f9202 100644 --- a/config/config.mainnet.yaml +++ b/config/config.mainnet.yaml @@ -106,6 +106,9 @@ features: providersFetch: enabled: true serviceUrl: 'https://api.multiversx.com' + assetsFetch: + enabled: true + assetesUrl: 'https://tools.multiversx.com/assets-cdn' image: width: 600 height: 600 diff --git a/config/config.testnet.yaml b/config/config.testnet.yaml index dd4bb8984..13f244dc4 100644 --- a/config/config.testnet.yaml +++ b/config/config.testnet.yaml @@ -105,6 +105,9 @@ features: providersFetch: enabled: true serviceUrl: 'https://testnet-api.multiversx.com' + assetsFetch: + enabled: true + assetesUrl: 'https://tools.multiversx.com/assets-cdn' image: width: 600 height: 600 diff --git a/src/common/api-config/api.config.service.ts b/src/common/api-config/api.config.service.ts index beff35a77..a1e79e5ec 100644 --- a/src/common/api-config/api.config.service.ts +++ b/src/common/api-config/api.config.service.ts @@ -866,6 +866,14 @@ export class ApiConfigService { return deepHistoryUrl; } + isAssetsCdnFeatureEnabled(): boolean { + return this.configService.get('features.assetsFetch.enabled') ?? false; + } + + getAssetsCdnUrl(): string { + return this.configService.get('features.assetsFetch.assetesUrl') ?? 'https://tools.multiversx.com/assets-cdn'; + } + isTokensFetchFeatureEnabled(): boolean { return this.configService.get('features.tokensFetch.enabled') ?? false; } diff --git a/src/common/assets/assets.service.ts b/src/common/assets/assets.service.ts index 601e544c4..bc3882366 100644 --- a/src/common/assets/assets.service.ts +++ b/src/common/assets/assets.service.ts @@ -1,131 +1,28 @@ import { Injectable } from "@nestjs/common"; -import simpleGit, { SimpleGit, SimpleGitOptions } from 'simple-git'; import { CacheInfo } from "src/utils/cache.info"; import { TokenAssets } from "src/common/assets/entities/token.assets"; -import { ApiConfigService } from "../api-config/api.config.service"; import { AccountAssets } from "./entities/account.assets"; import { CacheService } from "@multiversx/sdk-nestjs-cache"; -import { FileUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; -import { ApiUtils } from "@multiversx/sdk-nestjs-http"; import { MexPair } from "src/endpoints/mex/entities/mex.pair"; import { Identity } from "src/endpoints/identities/entities/identity"; import { MexFarm } from "src/endpoints/mex/entities/mex.farm"; import { MexSettings } from "src/endpoints/mex/entities/mex.settings"; import { DnsContracts } from "src/utils/dns.contracts"; -import { NftRankAlgorithm } from "./entities/nft.rank.algorithm"; import { NftRank } from "./entities/nft.rank"; import { MexStakingProxy } from "src/endpoints/mex/entities/mex.staking.proxy"; import { Provider } from "src/endpoints/providers/entities/provider"; - -const rimraf = require("rimraf"); -const path = require('path'); -const fs = require('fs'); +import { ApiService } from "@multiversx/sdk-nestjs-http"; +import { ApiConfigService } from "../api-config/api.config.service"; +import { KeybaseIdentity } from "../keybase/entities/keybase.identity"; @Injectable() export class AssetsService { - private readonly logger = new OriginLogger(AssetsService.name); - constructor( - private readonly cachingService: CacheService, private readonly apiConfigService: ApiConfigService, + private readonly apiService: ApiService, + private readonly cachingService: CacheService, ) { } - checkout(): Promise { - const localGitPath = 'dist/repos/assets'; - const logger = this.logger; - return new Promise((resolve, reject) => { - rimraf(localGitPath, function () { - logger.log("done deleting"); - - const options: Partial = { - baseDir: process.cwd(), - binary: 'git', - maxConcurrentProcesses: 6, - }; - - // when setting all options in a single object - const git: SimpleGit = simpleGit(options); - - // eslint-disable-next-line @typescript-eslint/no-floating-promises - git.outputHandler((_, stdout, stderr) => { - stdout.pipe(process.stdout); - stderr.pipe(process.stderr); - - stdout.on('data', (data) => { - // Print data - logger.log(data.toString('utf8')); - }); - }).clone('https://github.com/multiversx/mx-assets.git', localGitPath, undefined, (err) => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); - }); - } - - private readTokenAssetDetails(tokenIdentifier: string, assetPath: string): TokenAssets { - const infoPath = path.join(assetPath, 'info.json'); - const info = JSON.parse(fs.readFileSync(infoPath)); - - return new TokenAssets({ - ...info, - pngUrl: this.getImageUrl(tokenIdentifier, 'logo.png'), - svgUrl: this.getImageUrl(tokenIdentifier, 'logo.svg'), - }); - } - - private readTokenRanks(assetPath: string): NftRank[] | undefined { - const ranksPath = path.join(assetPath, 'ranks.json'); - if (fs.existsSync(ranksPath)) { - return JSON.parse(fs.readFileSync(ranksPath)); - } - - return undefined; - } - - private readAccountAssets(path: string): AccountAssets { - const jsonContents = fs.readFileSync(path); - const json = JSON.parse(jsonContents); - - return ApiUtils.mergeObjects(new AccountAssets(), json); - } - - private getImageUrl(tokenIdentifier: string, name: string) { - if (['mainnet', 'devnet', 'testnet'].includes(this.apiConfigService.getNetwork())) { - return `${this.apiConfigService.getExternalMediaUrl()}/tokens/asset/${tokenIdentifier}/${name}`; - } - - return `https://raw.githubusercontent.com/multiversx/mx-assets/master/${this.apiConfigService.getNetwork()}/tokens/${tokenIdentifier}/${name}`; - } - - private getTokenAssetsPath() { - return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('tokens')); - } - - private getAccountAssetsPath() { - return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('accounts')); - } - - getIdentityAssetsPath() { - return path.join(process.cwd(), 'dist/repos/assets', this.getRelativePath('identities')); - } - - getIdentityInfoJsonPath(identity: string): string { - return path.join(this.getIdentityAssetsPath(), identity, 'info.json'); - } - - private getRelativePath(name: string): string { - const network = this.apiConfigService.getNetwork(); - if (network !== 'mainnet') { - return path.join(network, name); - } - - return name; - } - async getAllTokenAssets(): Promise<{ [key: string]: TokenAssets }> { return await this.cachingService.getOrSet( CacheInfo.TokenAssets.key, @@ -134,24 +31,20 @@ export class AssetsService { ); } - getAllTokenAssetsRaw(): { [key: string]: TokenAssets } { - const tokensPath = this.getTokenAssetsPath(); - if (!fs.existsSync(tokensPath)) { + async getAllTokenAssetsRaw(): Promise<{ [key: string]: TokenAssets }> { + if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) { return {}; } - const tokenIdentifiers = FileUtils.getDirectories(tokensPath); + const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl(); + const network = this.apiConfigService.getNetwork(); + + const { data: assetsRaw } = await this.apiService.get(`${assetsCdnUrl}/${network}/tokens`); - // for every folder, create a TokenAssets entity with the contents of info.json and the urls from github const assets: { [key: string]: TokenAssets } = {}; - for (const tokenIdentifier of tokenIdentifiers) { - const tokenPath = path.join(tokensPath, tokenIdentifier); - try { - assets[tokenIdentifier] = this.readTokenAssetDetails(tokenIdentifier, tokenPath); - } catch (error) { - this.logger.error(`An error occurred while reading assets for token with identifier '${tokenIdentifier}'`); - this.logger.error(error); - } + for (const asset of assetsRaw) { + const { identifier, ...details } = asset; + assets[identifier] = new TokenAssets(details); } return assets; @@ -159,7 +52,6 @@ export class AssetsService { async getCollectionRanks(identifier: string): Promise { const allCollectionRanks = await this.getAllCollectionRanks(); - return allCollectionRanks[identifier]; } @@ -172,19 +64,23 @@ export class AssetsService { } async getAllCollectionRanksRaw(): Promise<{ [key: string]: NftRank[] }> { - const allTokenAssets = await this.getAllTokenAssets(); + if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) { + return {}; + } + + const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl(); + const network = this.apiConfigService.getNetwork(); + + const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/tokens`); const result: { [key: string]: NftRank[] } = {}; - const assetsPath = this.getTokenAssetsPath(); - - for (const identifier of Object.keys(allTokenAssets)) { - const assets = allTokenAssets[identifier]; - if (assets.preferredRankAlgorithm === NftRankAlgorithm.custom) { - const tokenAssetsPath = path.join(assetsPath, identifier); - const ranks = this.readTokenRanks(tokenAssetsPath); - if (ranks) { - result[identifier] = ranks; - } + + for (const asset of assets) { + if (asset.ranks && asset.ranks.length > 0) { + result[asset.identifier] = asset.ranks.map((rank: any) => new NftRank({ + identifier: rank.identifier, + rank: rank.rank, + })); } } @@ -199,37 +95,23 @@ export class AssetsService { ); } - getAllAccountAssetsRaw(providers?: Provider[], identities?: Identity[], pairs?: MexPair[], farms?: MexFarm[], mexSettings?: MexSettings, stakingProxies?: MexStakingProxy[]): { [key: string]: AccountAssets } { - const accountAssetsPath = this.getAccountAssetsPath(); - if (!fs.existsSync(accountAssetsPath)) { + async getAllAccountAssetsRaw(providers?: Provider[], identities?: Identity[], pairs?: MexPair[], farms?: MexFarm[], mexSettings?: MexSettings, stakingProxies?: MexStakingProxy[]): Promise<{ [key: string]: AccountAssets }> { + if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) { return {}; } - const fileNames = FileUtils.getFiles(accountAssetsPath); - const allAssets: { [key: string]: AccountAssets } = {}; - for (const fileName of fileNames) { - if (fileName.includes(".gitkeep")) { - continue; - } - const assetsPath = path.join(accountAssetsPath, fileName); - const address = fileName.removeSuffix('.json'); - try { - const assets = this.readAccountAssets(assetsPath); - if (assets.icon) { - const relativePath = this.getRelativePath(`accounts/icons/${assets.icon}`); - assets.iconPng = `https://raw.githubusercontent.com/multiversx/mx-assets/master/${relativePath}.png`; - assets.iconSvg = `https://raw.githubusercontent.com/multiversx/mx-assets/master/${relativePath}.svg`; - - delete assets.icon; - } + const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl(); + const network = this.apiConfigService.getNetwork(); - allAssets[address] = assets; - } catch (error) { - this.logger.error(`An error occurred while reading assets for account with address '${address}'`); - this.logger.error(error); - } + const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/accounts`); + + const allAssets: { [key: string]: AccountAssets } = {}; + for (const asset of assets) { + const { address, ...details } = asset; + allAssets[address] = new AccountAssets(details); } + // Populate additional assets from other sources if available if (providers && identities) { for (const provider of providers) { const identity = identities.find(x => x.identity === provider.identity); @@ -304,13 +186,34 @@ export class AssetsService { } async getTokenAssets(tokenIdentifier: string): Promise { - // get the dictionary from the local cache const assets = await this.getAllTokenAssets(); - - // if the tokenIdentifier key exists in the dictionary, return the associated value, else undefined return assets[tokenIdentifier]; } + async getAllIdentitiesRaw(): Promise<{ [key: string]: KeybaseIdentity }> { + if (!this.apiConfigService.isAssetsCdnFeatureEnabled()) { + return {}; + } + + const assetsCdnUrl = this.apiConfigService.getAssetsCdnUrl(); + const network = this.apiConfigService.getNetwork(); + + const { data: assets } = await this.apiService.get(`${assetsCdnUrl}/${network}/identities`); + + const allAssets: { [key: string]: KeybaseIdentity } = {}; + for (const asset of assets) { + const { identity, ...details } = asset; + allAssets[identity] = new KeybaseIdentity(details); + } + + return allAssets; + } + + async getIdentityInfo(identity: string): Promise { + const allIdentities = await this.getAllIdentitiesRaw(); + return allIdentities[identity] || null; + } + createAccountAsset(name: string, tags: string[]): AccountAssets { return new AccountAssets({ name: name, diff --git a/src/common/assets/entities/nft.rank.ts b/src/common/assets/entities/nft.rank.ts index 7d4267c41..50df3875a 100644 --- a/src/common/assets/entities/nft.rank.ts +++ b/src/common/assets/entities/nft.rank.ts @@ -3,6 +3,12 @@ import { ApiProperty } from "@nestjs/swagger"; @ObjectType("NftRank", { description: "NFT rank object type" }) export class NftRank { + constructor(init?: Partial) { + if (init) { + Object.assign(this, init); + } + } + @Field(() => String, { description: 'NFT identifier' }) @ApiProperty({ type: String }) identifier: string = ''; diff --git a/src/common/keybase/keybase.service.ts b/src/common/keybase/keybase.service.ts index c7234ddd7..b5b25b97d 100644 --- a/src/common/keybase/keybase.service.ts +++ b/src/common/keybase/keybase.service.ts @@ -3,8 +3,6 @@ import { NodeService } from "src/endpoints/nodes/node.service"; import { ProviderService } from "src/endpoints/providers/provider.service"; import { KeybaseIdentity } from "./entities/keybase.identity"; import { CacheInfo } from "../../utils/cache.info"; -import fs from 'fs'; -import { readdir } from 'fs/promises'; import { AssetsService } from "../assets/assets.service"; import { CacheService } from "@multiversx/sdk-nestjs-cache"; import { AddressUtils, OriginLogger } from "@multiversx/sdk-nestjs-common"; @@ -25,13 +23,8 @@ export class KeybaseService { ) { } private async getDistinctIdentities(): Promise { - const dirContents = await readdir(this.assetsService.getIdentityAssetsPath(), { withFileTypes: true }); - - const identities = dirContents - .filter(dirent => dirent.isDirectory()) - .map(dirent => dirent.name); - - return identities; + const identities = await this.assetsService.getAllIdentitiesRaw(); + return Object.keys(identities); } async confirmIdentities(): Promise { @@ -52,8 +45,8 @@ export class KeybaseService { } } - getOwners(identity: string): string[] | undefined { - const info = this.readIdentityInfoFile(identity); + async getOwners(identity: string): Promise { + const info = await this.readIdentityInfo(identity); if (!info || !info.owners) { return undefined; } @@ -62,7 +55,7 @@ export class KeybaseService { } async confirmIdentity(identity: string, providerAddresses: string[], blsIdentityDict: Record, confirmations: Record): Promise { - const keys = this.getOwners(identity); + const keys = await this.getOwners(identity); if (!keys) { return; } @@ -118,18 +111,13 @@ export class KeybaseService { return null; } - private readIdentityInfoFile(identity: string): any { - const filePath = this.assetsService.getIdentityInfoJsonPath(identity); - if (!fs.existsSync(filePath)) { - return null; - } - - const info = JSON.parse(fs.readFileSync(filePath).toString()); - return info; + private async readIdentityInfo(identity: string): Promise { + const identityInfo = await this.assetsService.getIdentityInfo(identity); + return identityInfo; } getProfileFromAssets(identity: string): KeybaseIdentity | null { - const info = this.readIdentityInfoFile(identity); + const info = this.readIdentityInfo(identity); if (!info) { return null; } diff --git a/src/crons/cache.warmer/cache.warmer.service.ts b/src/crons/cache.warmer/cache.warmer.service.ts index 8b4063a67..789e340b2 100644 --- a/src/crons/cache.warmer/cache.warmer.service.ts +++ b/src/crons/cache.warmer/cache.warmer.service.ts @@ -245,8 +245,7 @@ export class CacheWarmerService { @Lock({ name: 'Token / account assets invalidations', verbose: true }) async handleTokenAssetsInvalidations() { - await this.assetsService.checkout(); - const assets = this.assetsService.getAllTokenAssetsRaw(); + const assets = await this.assetsService.getAllTokenAssetsRaw(); await this.invalidateKey(CacheInfo.TokenAssets.key, assets, CacheInfo.TokenAssets.ttl); await this.keybaseService.confirmIdentities();