Reset transferLast24h to 0 for accounts that have no transactions in the last 24h (#1355)

* reset transferLast24h to 0 for accounts that have no transactions in the last 24h

* fix lint errors

---------

Co-authored-by: cfaur09 <[email protected]>

Author: gabrielmatei

src/common/indexer/elastic/elastic.indexer.service.ts

@@ -404,7 +404,7 @@ export class ElasticIndexerService implements IndexerInterface {
     return await this.elasticService.getList('operations', 'hash', elasticQuery);
   }
 
-  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<any[]> {
+  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<any[]> {
     let elasticQuery = this.indexerHelper.buildAccountFilterQuery(filter);
     const sortOrder: ElasticSortOrder = !filter.order || filter.order === SortOrder.desc ? ElasticSortOrder.descending : ElasticSortOrder.ascending;
     const sort: AccountSort = filter.sort ?? AccountSort.balance;
@@ -429,6 +429,10 @@ export class ElasticIndexerService implements IndexerInterface {
 
     elasticQuery = elasticQuery.withPagination(queryPagination);
 
+    if (fields && fields.length > 0) {
+      elasticQuery = elasticQuery.withFields(fields);
+    }
+
     return await this.elasticService.getList('accounts', 'address', elasticQuery);
   }
 
@@ -976,6 +980,16 @@ export class ElasticIndexerService implements IndexerInterface {
     return await this.elasticService.getCount('scdeploys', elasticQuery);
   }
 
+  async getAddressesWithTransfersLast24h(): Promise<string[]> {
+    const elasticQuery = ElasticQuery.create()
+      .withFields(['address'])
+      .withPagination({ from: 0, size: 10000 })
+      .withMustExistCondition('api_transfersLast24h');
+
+    const result = await this.elasticService.getList('accounts', 'address', elasticQuery);
+    return result.map(x => x.address);
+  }
+
   async getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]> {
     const elasticQuery = this.indexerHelper.buildEventsFilter(filter)
       .withPagination(pagination)

src/common/indexer/indexer.interface.ts

@@ -104,7 +104,7 @@ export interface IndexerInterface {
 
   getAccount(address: string): Promise<Account>
 
-  getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<Account[]>
+  getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<Account[]>
 
   getAccountDeploys(pagination: QueryPagination, address: string): Promise<ScDeploy[]>
 
@@ -188,6 +188,8 @@ export interface IndexerInterface {
 
   getApplicationCount(filter: ApplicationFilter): Promise<number>
 
+  getAddressesWithTransfersLast24h(): Promise<string[]>
+
   getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]>
 
   getEvent(txHash: string): Promise<Events>

src/common/indexer/indexer.service.ts

@@ -227,8 +227,8 @@ export class IndexerService implements IndexerInterface {
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
-  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions): Promise<Account[]> {
-    return await this.indexerInterface.getAccounts(queryPagination, filter);
+  async getAccounts(queryPagination: QueryPagination, filter: AccountQueryOptions, fields?: string[]): Promise<Account[]> {
+    return await this.indexerInterface.getAccounts(queryPagination, filter, fields);
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
@@ -452,6 +452,10 @@ export class IndexerService implements IndexerInterface {
   }
 
   @LogPerformanceAsync(MetricsEvents.SetIndexerDuration)
+  async getAddressesWithTransfersLast24h(): Promise<string[]> {
+    return await this.indexerInterface.getAddressesWithTransfersLast24h();
+  }
+
   async getEvents(pagination: QueryPagination, filter: EventsFilter): Promise<Events[]> {
     return await this.indexerInterface.getEvents(pagination, filter);
   }

src/crons/cache.warmer/cache.warmer.service.ts

@@ -33,6 +33,7 @@ import { PoolService } from "src/endpoints/pool/pool.service";
 import * as JsonDiff from "json-diff";
 import { QueryPagination } from "src/common/entities/query.pagination";
 import { StakeService } from "src/endpoints/stake/stake.service";
+import { ApplicationMostUsed } from "src/endpoints/accounts/entities/application.most.used";
 
 @Injectable()
 export class CacheWarmerService {
@@ -377,21 +378,28 @@ export class CacheWarmerService {
   async handleUpdateAccountTransfersLast24h() {
     const batchSize = 100;
     const mostUsed = await this.accountService.getApplicationMostUsedRaw();
+    const mostUsedIndexedAccounts = await this.indexerService.getAddressesWithTransfersLast24h();
 
-    const batches = BatchUtils.splitArrayIntoChunks(mostUsed, batchSize);
+    const allAddressesToUpdate = [...mostUsed.map(item => item.address), ...mostUsedIndexedAccounts].distinct();
+    const mostUsedDictionary = mostUsed.toRecord<ApplicationMostUsed>(item => item.address);
+
+    const batches = BatchUtils.splitArrayIntoChunks(allAddressesToUpdate, batchSize);
     for (const batch of batches) {
       const accounts = await this.indexerService.getAccounts(
         new QueryPagination({ from: 0, size: batchSize }),
-        new AccountQueryOptions({ addresses: batch.map(item => item.address) }),
+        new AccountQueryOptions({ addresses: batch }),
+        ['address', 'api_transfersLast24h'],
       );
 
-      const accountsDictionary = accounts.toRecord<Account>(account => account.address);
+      const accountsDictionary = accounts.toRecord<Pick<Account, 'address' | 'api_transfersLast24h'>>(account => account.address);
+
+      for (const address of batch) {
+        const account = accountsDictionary[address];
+        const newTransfersLast24h = mostUsedDictionary[address]?.transfers24H ?? 0;
 
-      for (const item of batch) {
-        const account = accountsDictionary[item.address];
-        if (account && account.api_transfersLast24h !== item.transfers24H) {
-          this.logger.log(`Setting transferLast24h to ${item.transfers24H} for account with address '${item.address}'`);
-          await this.indexerService.setAccountTransfersLast24h(item.address, item.transfers24H);
+        if (account && account.api_transfersLast24h !== newTransfersLast24h) {
+          this.logger.log(`Setting transferLast24h to ${newTransfersLast24h} for account with address '${address}'`);
+          await this.indexerService.setAccountTransfersLast24h(address, newTransfersLast24h);
         }
       }
     }