API-72: first integration of chain simulator

.github/workflows/chain-simulator-e2e-tests.yml

@@ -52,8 +52,8 @@ jobs:
 
       - name: Run e2e tests
         run: npm run test:cs-e2e
-      
+
       - name: Stop API after tests
         run: |
           echo "Stopping the API..."
-          kill $(lsof -t -i:3001)
+          kill $(lsof -t -i:3001)

.github/workflows/load-tests.yml

@@ -1,8 +1,6 @@
 name: Load Tests
 
 on:
-  push:
-    branches: [main, development]
   pull_request:
     branches: [main, development]
 

.gitignore

@@ -41,4 +41,7 @@ lerna-debug.log*
 
 /src/plugins
 
-.env
+.env
+
+# CS Environment variables
+src/test/chain-simulator/config/.env

package.json

@@ -78,7 +78,8 @@
     "copy-e2e-mocked-mainnet-config:nix": "cp ./config/config.e2e-mocked.mainnet.yaml ./config/config.yaml",
     "copy-e2e-mainnet-config:windows": "copy .\\config\\config.e2e.mainnet.yaml .\\config\\config.yaml",
     "copy-e2e-mocked-mainnet-config:windows": "copy .\\config\\config.e2e-mocked.mainnet.yaml .\\config\\config.yaml",
-    "start-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" up -d --build"
+    "start-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" up -d --build",
+    "stop-chain-simulator": "docker compose -f \"src/test/chain-simulator/docker/docker-compose.yml\" down"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.54.0",
@@ -214,4 +215,4 @@
       "node_modules"
     ]
   }
-}
+}

src/common/assets/assets.service.ts

@@ -202,8 +202,7 @@ export class AssetsService {
 
     const allAssets: { [key: string]: KeybaseIdentity } = {};
     for (const asset of assets) {
-      const { identity, ...details } = asset;
-      allAssets[identity] = new KeybaseIdentity(details);
+      allAssets[asset.identity] = new KeybaseIdentity(asset);
     }
 
     return allAssets;

src/common/keybase/keybase.service.ts

@@ -97,12 +97,12 @@ export class KeybaseService {
 
   async confirmIdentityProfiles(): Promise<void> {
     const identities = await this.getDistinctIdentities();
-    const keybaseIdentities = identities.map(identity => this.getProfile(identity));
+    const keybaseIdentities = await Promise.all(identities.map(identity => this.getProfile(identity)));
     await this.cachingService.set(CacheInfo.IdentityProfilesKeybases.key, keybaseIdentities, CacheInfo.IdentityProfilesKeybases.ttl);
   }
 
-  getProfile(identity: string): KeybaseIdentity | null {
-    const keybaseLocal = this.getProfileFromAssets(identity);
+  async getProfile(identity: string): Promise<KeybaseIdentity | null> {
+    const keybaseLocal = await this.getProfileFromAssets(identity);
     if (keybaseLocal) {
       this.logger.log(`Got profile details from assets for identity '${identity}'`);
       return keybaseLocal;
@@ -116,8 +116,8 @@ export class KeybaseService {
     return identityInfo;
   }
 
-  getProfileFromAssets(identity: string): KeybaseIdentity | null {
-    const info = this.readIdentityInfo(identity);
+  async getProfileFromAssets(identity: string): Promise<KeybaseIdentity | null> {
+    const info = await this.readIdentityInfo(identity);
     if (!info) {
       return null;
     }

src/endpoints/accounts/account.controller.ts

@@ -191,7 +191,7 @@ export class AccountController {
   @UseInterceptors(DeepHistoryInterceptor)
   @ApiOperation({ summary: 'Account details', description: 'Returns account details for a given address' })
   @ApiQuery({ name: 'withGuardianInfo', description: 'Returns guardian data for a given address', required: false })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'timestamp', description: 'Retrieve entry from timestamp', required: false, type: Number })
   @ApiOkResponse({ type: AccountDetailed })
   async getAccountDetails(
@@ -240,6 +240,7 @@ export class AccountController {
   @ApiQuery({ name: 'from', description: 'Number of items to skip for the result set', required: false })
   @ApiQuery({ name: 'size', description: 'Number of items to retrieve', required: false })
   @ApiQuery({ name: 'type', description: 'Token type', required: false, enum: TokenType })
+  @ApiQuery({ name: 'subType', description: 'Token sub type', required: false, enum: NftSubType })
   @ApiQuery({ name: 'search', description: 'Search by collection identifier', required: false })
   @ApiQuery({ name: 'name', description: 'Search by token name', required: false })
   @ApiQuery({ name: 'identifier', description: 'Search by token identifier', required: false })
@@ -253,6 +254,7 @@ export class AccountController {
     @Query('from', new DefaultValuePipe(0), ParseIntPipe) from: number,
     @Query('size', new DefaultValuePipe(25), ParseIntPipe) size: number,
     @Query('type', new ParseEnumPipe(TokenType)) type?: TokenType,
+    @Query('subType', new ParseEnumPipe(NftSubType)) subType?: NftSubType,
     @Query('search') search?: string,
     @Query('name') name?: string,
     @Query('identifier') identifier?: string,
@@ -262,7 +264,7 @@ export class AccountController {
     @Query('mexPairType', new ParseEnumArrayPipe(MexPairType)) mexPairType?: MexPairType[],
   ): Promise<TokenWithBalance[]> {
     try {
-      return await this.tokenService.getTokensForAddress(address, new QueryPagination({ from, size }), new TokenFilter({ type, search, name, identifier, identifiers, includeMetaESDT, mexPairType }));
+      return await this.tokenService.getTokensForAddress(address, new QueryPagination({ from, size }), new TokenFilter({ type, subType, search, name, identifier, identifiers, includeMetaESDT, mexPairType }));
     } catch (error) {
       this.logger.error(`Error in getAccountTokens for address ${address}`);
       this.logger.error(error);
@@ -613,7 +615,7 @@ export class AccountController {
   @ApiQuery({ name: 'withScamInfo', description: 'Include scam info in the response', required: false, type: Boolean })
   @ApiQuery({ name: 'computeScamInfo', description: 'Compute scam info in the response', required: false, type: Boolean })
   @ApiQuery({ name: 'excludeMetaESDT', description: 'Exclude NFTs of type "MetaESDT" in the response', required: false, type: Boolean })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'isScam', description: 'Filter by scam status', required: false, type: Boolean })
   @ApiQuery({ name: 'scamType', description: 'Filter by type (scam/potentialScam)', required: false })
   @ApiQuery({ name: 'timestamp', description: 'Retrieve entry from timestamp', required: false, type: Number })
@@ -747,7 +749,7 @@ export class AccountController {
   @Get("/accounts/:address/nfts/:nft")
   @UseInterceptors(DeepHistoryInterceptor)
   @ApiOperation({ summary: 'Account NFT/SFT token details', description: 'Returns details about a specific fungible token for a given address' })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'extract', description: 'Extract a specific field', required: false })
   @ApiQuery({ name: 'timestamp', description: 'Retrieve entry from timestamp', required: false, type: Number })
   @ApiOkResponse({ type: NftAccount })
@@ -849,7 +851,7 @@ export class AccountController {
   @ApiQuery({ name: 'status', description: 'Status of the transaction (success / pending / invalid / fail)', required: false, enum: TransactionStatus })
   @ApiQuery({ name: 'function', description: 'Filter transactions by function name', required: false })
   @ApiQuery({ name: 'order', description: 'Sort order (asc/desc)', required: false, enum: SortOrder })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'before', description: 'Before timestamp', required: false })
   @ApiQuery({ name: 'after', description: 'After timestamp', required: false })
   @ApiQuery({ name: 'round', description: 'Round number', required: false })
@@ -984,7 +986,7 @@ export class AccountController {
   @ApiQuery({ name: 'before', description: 'Before timestamp', required: false })
   @ApiQuery({ name: 'after', description: 'After timestamp', required: false })
   @ApiQuery({ name: 'round', description: 'Round number', required: false })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'relayer', description: 'Address of the relayer', required: false })
   @ApiQuery({ name: 'withScamInfo', description: 'Returns scam information', required: false, type: Boolean })
   @ApiQuery({ name: 'withUsername', description: 'Integrates username in assets for all addresses present in the transactions', required: false, type: Boolean })

src/endpoints/nfts/entities/nft.sub.type.ts

@@ -8,6 +8,7 @@ export enum NftSubType {
   DynamicNonFungibleESDT = 'DynamicNonFungibleESDT',
   DynamicSemiFungibleESDT = 'DynamicSemiFungibleESDT',
   DynamicMetaESDT = 'DynamicMetaESDT',
+  None = '',
 }
 
 registerEnumType(NftSubType, {
@@ -35,5 +36,8 @@ registerEnumType(NftSubType, {
     DynamicMetaESDT: {
       description: 'Dynamic meta ESDT NFT type.',
     },
+    None: {
+      description: '',
+    },
   },
 });

src/endpoints/tokens/entities/token.filter.ts

@@ -3,6 +3,7 @@ import { TokenType } from "src/common/indexer/entities";
 import { TokenSort } from "./token.sort";
 import { MexPairType } from "src/endpoints/mex/entities/mex.pair.type";
 import { TokenAssetsPriceSourceType } from "src/common/assets/entities/token.assets.price.source.type";
+import { NftSubType } from "../../nfts/entities/nft.sub.type";
 
 export class TokenFilter {
   constructor(init?: Partial<TokenFilter>) {
@@ -11,6 +12,8 @@ export class TokenFilter {
 
   type?: TokenType;
 
+  subType?: NftSubType;
+
   search?: string;
 
   name?: string;

src/endpoints/tokens/entities/token.ts

@@ -4,6 +4,7 @@ import { TokenType } from "src/common/indexer/entities";
 import { TokenAssets } from "../../../common/assets/entities/token.assets";
 import { MexPairType } from "src/endpoints/mex/entities/mex.pair.type";
 import { TokenOwnersHistory } from "./token.owner.history";
+import { NftSubType } from "../../nfts/entities/nft.sub.type";
 
 export class Token {
   constructor(init?: Partial<Token>) {
@@ -13,6 +14,9 @@ export class Token {
   @ApiProperty({ enum: TokenType })
   type: TokenType = TokenType.FungibleESDT;
 
+  @ApiProperty({ enum: NftSubType })
+  subType: NftSubType = NftSubType.None;
+
   @ApiProperty({ type: String })
   identifier: string = '';
 

src/endpoints/tokens/token.controller.ts

@@ -341,7 +341,7 @@ export class TokenController {
   }
 
   @Get("/tokens/:identifier/roles/:address")
-  @ApiOperation({ summary: 'Token address roles', description: 'Returns roles detalils for a specific address of a given token', deprecated: true })
+  @ApiOperation({ summary: 'Token address roles', description: 'Returns roles details for a specific address of a given token', deprecated: true })
   @ApiOkResponse({ type: TokenRoles })
   @ApiNotFoundResponse({ description: 'Token not found' })
   async getTokenRolesForAddress(
@@ -379,7 +379,7 @@ export class TokenController {
   @ApiQuery({ name: 'before', description: 'Before timestamp', required: false })
   @ApiQuery({ name: 'after', description: 'After timestamp', required: false })
   @ApiQuery({ name: 'round', description: 'Filter by round number', required: false })
-  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false })
+  @ApiQuery({ name: 'fields', description: 'List of fields to filter by', required: false, isArray: true, style: 'form', explode: false })
   @ApiQuery({ name: 'withScamInfo', description: 'Returns scam information', required: false, type: Boolean })
   @ApiQuery({ name: 'withUsername', description: 'Integrates username in assets for all addresses present in the transactions', required: false, type: Boolean })
   @ApiQuery({ name: 'withBlockInfo', description: 'Returns sender / receiver block details', required: false, type: Boolean })
@@ -435,7 +435,7 @@ export class TokenController {
   }
 
   @Get("/tokens/:identifier/transfers/count")
-  @ApiOperation({ summary: 'Account transfer count', description: 'Return total count of tranfers triggerred by a user account (type = Transaction), as well as transfers triggerred by smart contracts (type = SmartContractResult)' })
+  @ApiOperation({ summary: 'Account transfer count', description: 'Return total count of transfers triggerred by a user account (type = Transaction), as well as transfers triggerred by smart contracts (type = SmartContractResult)' })
   @ApiOkResponse({ type: Number })
   @ApiQuery({ name: 'sender', description: 'Address of the transfer sender', required: false })
   @ApiQuery({ name: 'receiver', description: 'Search by multiple receiver addresses, comma-separated', required: false })

src/endpoints/tokens/token.service.ts

@@ -42,10 +42,14 @@ import { TransferService } from "../transfers/transfer.service";
 import { MexPairService } from "../mex/mex.pair.service";
 import { MexPairState } from "../mex/entities/mex.pair.state";
 import { MexTokenType } from "../mex/entities/mex.token.type";
+import { NftSubType } from "../nfts/entities/nft.sub.type";
 
 @Injectable()
 export class TokenService {
   private readonly logger = new OriginLogger(TokenService.name);
+  private readonly nftSubTypes = [NftSubType.DynamicNonFungibleESDT, NftSubType.DynamicMetaESDT, NftSubType.NonFungibleESDTv2, NftSubType.DynamicSemiFungibleESDT];
+  private readonly egldIdentifierInMultiTransfer = 'EGLD-000000';
+
   constructor(
     private readonly esdtService: EsdtService,
     private readonly indexerService: IndexerService,
@@ -113,7 +117,7 @@ export class TokenService {
   }
 
   async getTokens(queryPagination: QueryPagination, filter: TokenFilter): Promise<TokenDetailed[]> {
-    const { from, size } = queryPagination;
+    const {from, size} = queryPagination;
 
     let tokens = await this.getFilteredTokens(filter);
 
@@ -123,7 +127,9 @@ export class TokenService {
       this.applyTickerFromAssets(token);
     }
 
-    return tokens.map(item => ApiUtils.mergeObjects(new TokenDetailed(), item));
+   return tokens
+    .map(item => ApiUtils.mergeObjects(new TokenDetailed(), item))
+    .filter(t => t.identifier !== this.egldIdentifierInMultiTransfer);
   }
 
   applyTickerFromAssets(token: Token) {
@@ -141,6 +147,10 @@ export class TokenService {
       tokens = tokens.filter(token => token.type === filter.type);
     }
 
+    if (filter.subType) {
+      tokens = tokens.filter(token => token.subType.toString() === filter.subType?.toString());
+    }
+
     if (filter.search) {
       const searchLower = filter.search.toLowerCase();
 
@@ -350,11 +360,11 @@ export class TokenService {
     if (TokenUtils.isNft(identifier)) {
       const nftData = await this.gatewayService.getAddressNft(address, identifier);
 
-      tokenWithBalance = new TokenDetailedWithBalance({ ...token, ...nftData });
+      tokenWithBalance = new TokenDetailedWithBalance({...token, ...nftData});
     } else {
       const esdtData = await this.gatewayService.getAddressEsdt(address, identifier);
 
-      tokenWithBalance = new TokenDetailedWithBalance({ ...token, ...esdtData });
+      tokenWithBalance = new TokenDetailedWithBalance({...token, ...esdtData});
     }
 
     // eslint-disable-next-line require-await
@@ -398,11 +408,36 @@ export class TokenService {
         continue;
       }
 
+      if (esdt.type && this.nftSubTypes.includes(esdt.type)) {
+        switch (esdt.type as NftSubType) {
+          case NftSubType.DynamicNonFungibleESDT:
+          case NftSubType.NonFungibleESDTv2:
+            esdt.type = NftSubType.NonFungibleESDT;
+            esdt.subType = esdt.type;
+            break;
+          case NftSubType.DynamicMetaESDT:
+            esdt.type = NftType.MetaESDT;
+            esdt.subType = NftSubType.DynamicMetaESDT;
+            break;
+          case NftSubType.DynamicSemiFungibleESDT:
+            esdt.type = NftType.SemiFungibleESDT;
+            esdt.subType = NftSubType.DynamicSemiFungibleESDT;
+            break;
+          default:
+            esdt.subType = NftSubType.None;
+            break;
+        }
+      }
+
       const tokenWithBalance = {
         ...token,
         ...esdt,
       };
 
+      if (esdt.type === '') { // empty type can come from gateway
+        tokenWithBalance.type = token.type;
+      }
+
       tokensWithBalance.push(tokenWithBalance);
     }
 
@@ -658,8 +693,6 @@ export class TokenService {
     return result;
   }
 
-
-
   private async getLogo(identifier: string): Promise<TokenLogo | undefined> {
     const assets = await this.assetsService.getTokenAssets(identifier);
     if (!assets) {
@@ -712,7 +745,7 @@ export class TokenService {
     return await this.cachingService.getOrSet(
       CacheInfo.AllEsdtTokens.key,
       async () => await this.getAllTokensRaw(),
-      CacheInfo.AllEsdtTokens.ttl
+      CacheInfo.AllEsdtTokens.ttl,
     );
   }
 
@@ -746,6 +779,7 @@ export class TokenService {
     for (const collection of collections) {
       tokens.push(new TokenDetailed({
         type: TokenType.MetaESDT,
+        subType: collection.subType,
         identifier: collection.collection,
         name: collection.name,
         timestamp: collection.timestamp,
@@ -807,6 +841,20 @@ export class TokenService {
       token => token.transactions ?? 0,
     );
 
+    const egldToken = new TokenDetailed({
+      identifier: this.egldIdentifierInMultiTransfer,
+      name: 'EGLD',
+      type: TokenType.FungibleESDT,
+      assets: await this.assetsService.getTokenAssets(this.egldIdentifierInMultiTransfer),
+      decimals: 18,
+      isLowLiquidity: false,
+      price: await this.dataApiService.getEgldPrice(),
+      supply: '0',
+      circulatingSupply: '0',
+      marketCap: 0,
+    });
+    tokens = [...tokens, egldToken];
+
     return tokens;
   }
 
@@ -921,9 +969,9 @@ export class TokenService {
 
   private async getTotalTransactions(token: TokenDetailed): Promise<{ count: number, lastUpdatedAt: number } | undefined> {
     try {
-      const count = await this.transactionService.getTransactionCount(new TransactionFilter({ tokens: [token.identifier, ...token.assets?.extraTokens ?? []] }));
+      const count = await this.transactionService.getTransactionCount(new TransactionFilter({tokens: [token.identifier, ...token.assets?.extraTokens ?? []]}));
 
-      return { count, lastUpdatedAt: new Date().getTimeInSeconds() };
+      return {count, lastUpdatedAt: new Date().getTimeInSeconds()};
     } catch (error) {
       this.logger.error(`An unhandled error occurred when getting transaction count for token '${token.identifier}'`);
       this.logger.error(error);