Improve memoization logic

api/Dockerfile

@@ -19,6 +19,7 @@ RUN npm run build
 FROM node:20.12.2-bullseye AS run
 
 ARG ENV=dev
+ENV ENV=${ENV}
 ENV NODE_ENV=${ENV}
 ARG NODE_OPTIONS
 ENV NODE_OPTIONS=${NODE_OPTIONS}

api/src/@utils/async/promise.ts

@@ -94,17 +94,26 @@ export async function promiseRetry<T>(func: () => Promise<T>, maxRetries: number
     }
 }
 
-function isPromise<T>(value: T | Promise<T>): value is Promise<T> {
+export type TypedFunction<T = any> = (...args: any[]) => any | Promise<T>;
+export type TypedFunctionWrapper<T extends TypedFunction> = ((...args: Parameters<T>) => ReturnType<T>);
+export type TypedSyncFunction<T = any> = (...args: any[]) => T;
+export type TypedAsyncFunction<T = any> = (...args: any[]) => Promise<T>;
+
+export function isPromise<T>(value: T | Promise<T>): value is Promise<T> {
     return value instanceof Promise;
 }
 
+export function isAsyncFunction(func: TypedFunction): func is TypedAsyncFunction {
+    return func.constructor.name == 'AsyncFunction';
+}
+
 // Receives a value that could be already resolved or yet a Promise,
 // and calls the callback with the resolved value,
 // returning the resolved result or a new promise
 // export function runOrResolve<T, U>(value: T, func: (resolved: T) => U): U;
 // export function runOrResolve<T, U>(value: Promise<T>, func: (resolved: T) => U): Promise<U>;
-export function runOrResolve<T, U>(value: T | Promise<T>, callback: (resolved: T) => U): U | Promise<U> {
+export function runOrResolve<T, U>(value: T | Promise<T>, callback: (resolved: T) => U, errCallback?: (rejected: any) => any): U | Promise<U> {
     return isPromise(value)
-        ? value.then(resolved => callback(resolved as T)) 
+        ? value.then(callback).catch(err => errCallback?.(err) ?? (err => { throw err; })(err))
         : callback(value as T);
 }

api/src/@utils/cache/memoize.ts

@@ -1,17 +1,14 @@
-import { jsonDateReviver, readFileSync, runOrResolve, tryParseJson, tryStringifyJson, writeFileSync } from '@/@utils';
+import { jsonDateReviver, readFileSync, runOrResolve, tryParseJson, tryStringifyJson, TypedFunction, TypedFunctionWrapper, writeFileSync } from '@/@utils';
 import { createHash } from 'crypto';
 import * as fs from 'fs';
 import { isEqual, merge } from 'lodash';
-import { RFC_2822 } from 'moment-timezone';
 import { tmpdir } from 'os';
 import * as path from 'path';
 
 // TYPES
 
-export type TypedFunction = (...args: any[]) => any;
-
 export type MemoizedProps = { _memoizeCache: MemoizeCache<any, any> };
-export type Memoized<T extends TypedFunction> = ((...args: Parameters<T>) => ReturnType<T>) & MemoizedProps;
+export type Memoized<F extends TypedFunction> = TypedFunctionWrapper<F> & MemoizedProps;
 
 export interface MemoizeConfig {
   funcKey?: (config: MemoizeConfig) => string;
@@ -38,25 +35,25 @@ export function isMemoizeCacheType(value: any): value is MemoizeCacheType {
   return Object.values(MemoizeCacheType).includes(value as MemoizeCacheType);
 }
 
-export interface MemoizeCacheItem<T extends TypedFunction> {
+export interface MemoizeCacheItem<F extends TypedFunction> {
   date: Date,
-  value: ReturnType<T>,
+  value: ReturnType<F>,
 }
 
-export type MemoizeCacheMap<T extends TypedFunction> = Record<string, MemoizeCacheItem<T>>;
+export type MemoizeCacheMap<F extends TypedFunction> = Record<string, MemoizeCacheItem<F>>;
 
-export interface MemoizeCache<T extends TypedFunction, C> {
+export interface MemoizeCache<F extends TypedFunction, C> {
   disabled?: boolean,
   config?: MemoizeConfig['cacheConfig'],
   cache: C,
   state: any,
-  memoizedFunc?: Memoized<ReturnType<T>>,
+  memoizedFunc?: Memoized<ReturnType<F>>,
   init: () => void,
-  get: (key: string) => MemoizeCacheItem<T> | undefined,
-  set: (key: string, value: MemoizeCacheItem<T> | Promise<MemoizeCacheItem<T>>) => void,
+  get: (key: string) => MemoizeCacheItem<F> | undefined,
+  set: (key: string, value: MemoizeCacheItem<F> | Promise<MemoizeCacheItem<F>>) => void,
   delete: (key: string) => void,
   flush: () => void,
-  invalidate: (predicate: (key: string, value: MemoizeCacheItem<T>) => boolean) => void,
+  invalidate: (predicate: (key: string, value: MemoizeCacheItem<F>) => boolean) => void,
 }
 
 // CONSTS
@@ -68,8 +65,8 @@ export const globalMemoizeConfig: Partial<MemoizeConfig> = { // HOW-TO: globalMe
   }
 };
 
-const memoizeMemory = <T extends TypedFunction>(config: MemoizeConfig) => {
-  const memoryCache: MemoizeCache<T, MemoizeCacheMap<T>> = {
+const memoizeMemory = <F extends TypedFunction>(config: MemoizeConfig) => {
+  const memoryCache: MemoizeCache<F, MemoizeCacheMap<F>> = {
     config: config.cacheConfig,
     cache: {},
     state: {},
@@ -94,7 +91,7 @@ const memoizeMemory = <T extends TypedFunction>(config: MemoizeConfig) => {
   return memoryCache;
 };
 
-const memoizeStorage = <T extends TypedFunction>(config: MemoizeConfig) => {
+const memoizeStorage = <F extends TypedFunction>(config: MemoizeConfig) => {
   // TODO: do not load all to memory
 
   const getFileInfo = (cachePath: string) => {
@@ -109,14 +106,14 @@ const memoizeStorage = <T extends TypedFunction>(config: MemoizeConfig) => {
   };
 
   // Create + Fetch + Modify + Save
-  const syncCache = (callback?: (cacheObj: MemoizeCacheMap<T>) => MemoizeCacheMap<T> | Promise<MemoizeCacheMap<T>>) => {
+  const syncCache = (callback?: (cacheObj: MemoizeCacheMap<F>) => MemoizeCacheMap<F> | Promise<MemoizeCacheMap<F>>) => {
     const fileInfoKey = '_fileInfo';
     const fileInfo = getFileInfo(storageCache.config.cachePath);
     const oldFileInfo = storageCache.state[fileInfoKey];
     const fileChanged = fileInfo && (oldFileInfo == null || oldFileInfo.hash !== fileInfo.hash);
 
     const prevCache = fileChanged
-      ? tryParseJson<MemoizeCacheMap<T>>(readFileSync(storageCache.config.cachePath), jsonDateReviver)
+      ? tryParseJson<MemoizeCacheMap<F>>(readFileSync(storageCache.config.cachePath), jsonDateReviver)
       : storageCache.cache;
     const newCache = callback?.(prevCache) ?? prevCache;
 
@@ -127,18 +124,18 @@ const memoizeStorage = <T extends TypedFunction>(config: MemoizeConfig) => {
       const newFileInfo = getFileInfo(storageCache.config.cachePath);
       storageCache.state[fileInfoKey] = newFileInfo;
       return storageCache.cache;
-    });
+    }, err => err);
   }
 
   if (config.cacheConfig.cacheDir == null && config.cacheConfig.cachePath == null) throw new Error('Invalid cacheDir (null)');
   const defaultConfig: MemoizeCacheConfig = { cachePath: path.join(config.cacheConfig.cacheDir, `memoize-${config.funcKey(config)}.json`) };
 
-  const storageCache: MemoizeCache<T, MemoizeCacheMap<T>> = {
+  const storageCache: MemoizeCache<F, MemoizeCacheMap<F>> = {
     config: merge(defaultConfig, config.cacheConfig),
     cache: {},
     state: {},
     init: () => {
-      syncCache();
+      syncCache(); 
     },
     get: key => {
       const cache = syncCache() as object;
@@ -175,14 +172,14 @@ const cacheTypes: Record<MemoizeCacheType, (config: MemoizeConfig) => MemoizeCac
 
 // FUNCTIONS
 
-export function memoize<T extends (...args: any[]) => Promise<any>>(func: Memoized<T>, config: MemoizeConfig): Memoized<T> {
+export function memoize<F extends TypedFunction>(func: Memoized<F>, config: MemoizeConfig): Memoized<F> {
   config = merge(globalMemoizeConfig, config);
 
   if (config.funcKey == null) throw new Error('Invalid funcKey');
 
   const cache = isMemoizeCacheType(config.cacheType) ? cacheTypes[config.cacheType](config) : config.cacheType;
 
-  const memoized: Memoized<T> = function (...args: Parameters<T>): ReturnType<T> {
+  const memoized: Memoized<F> = function (...args: Parameters<F>): ReturnType<F> {
     if (cache.disabled) {
       config.onCall?.(config, args, cache);
       return func(...args);
@@ -195,8 +192,8 @@ export function memoize<T extends (...args: any[]) => Promise<any>>(func: Memoiz
     const cacheEntry = cache.get(itemKey);
 
     if (cacheEntry !== undefined) {
-      const cacheAge = (Date.now() - cacheEntry.date.getTime()) / 1000;
-      const validEntry = config.cacheConfig.maxAge ? cacheAge < config.cacheConfig.maxAge : true;
+      const cacheAge = cacheEntry.date != null ? (Date.now() - cacheEntry.date.getTime()) / 1000 : null;
+      const validEntry = cacheAge != null && (config.cacheConfig.maxAge ? cacheAge < config.cacheConfig.maxAge : true);
       if (validEntry) return cacheEntry.value;
       else cache.delete(itemKey);
     }
@@ -208,7 +205,7 @@ export function memoize<T extends (...args: any[]) => Promise<any>>(func: Memoiz
     return result;
   };
 
-  memoized._memoizeCache = cache; // HOW-TO: (this.myFunc as Memoized<any>)._memoizeCache.flush()
+  memoized._memoizeCache = cache;
   cache.memoizedFunc = memoized;
 
   return memoized;
@@ -228,7 +225,7 @@ export function Memoize(config?: MemoizeConfig) { // use with "@Memoize()"
         config._instance = this;
         config.funcKey ??= () => `${config._instance?.constructor?.name}:${config._method.value?.name}`;
 
-        const memoized = memoize(config._method.value.bind(this), config);
+        const memoized = memoize(config._method.value.bind(config._instance), config).bind(config._instance);
 
         Object.defineProperty(this, name, {
           value: memoized,

api/src/core/services/BaseAssetSgsService.ts

@@ -70,7 +70,7 @@ export abstract class BaseAssetSgsService<T extends AssetData> extends BaseAsset
         let data = await promiseRetry(
             () => HttpService.get(this.jsonUrl, { responseType: 'text' }).then(r => r.data),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         data = tryParseJson<AssetSgsDto[]>(data, undefined, false);

api/src/gov-bond/services/gov-bond-day-last-td.service.ts

@@ -113,7 +113,7 @@ export class GovBondDayLastTdService extends BaseAssetService {
         const data = await promiseRetry(() =>
             HttpService.get<GovBondDayLastTdDto>(this.jsonUrl, { httpsAgent }).then(r => r.data),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         return data;

api/src/gov-bond/services/gov-bond-day-sisweb.service.ts

@@ -139,7 +139,7 @@ export class GovBondDaySiswebService extends BaseAssetService {
                     const file = await promiseRetry(
                         () => HttpService.get(e.asset.url!, { responseType: 'arraybuffer' }).then(r => Buffer.from(r.data, 'binary')),
                         3,
-                        err => this.logger.log(`Retry Error: ${err}`)
+                        err => this.logger.warn(`Retry Error: ${err}`)
                     );
 
                     assetsData.push({
@@ -178,13 +178,13 @@ export class GovBondDaySiswebService extends BaseAssetService {
                 validateStatus: status => status >= 200 && status < 303,
             }).then(r => r.headers["set-cookie"]),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         html = await promiseRetry(
             () => HttpService.get(this.indexUrl, { headers: { 'Cookie': cookie[0] } }).then(r => r.data),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         const htmlMain = html.match(/<div class="bl-body">[\s\S]*?<\/div>/)?.[0];

api/src/gov-bond/services/gov-bond-day-transparente.service.ts

@@ -128,7 +128,7 @@ export class GovBondDayTransparenteService extends BaseAssetService {
         const file = await promiseRetry(
             () => HttpService.get(this.csvUrl, { responseType: 'arraybuffer' }).then(r => Buffer.from(r.data, 'binary')),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         const assetsData: GovBondDayTransparenteDto = {};

api/src/ipca/services/ipca-month-ipea.service.ts

@@ -68,7 +68,7 @@ export class IpcaMonthIpeaService extends BaseAssetService {
         const data = await promiseRetry(
             () => HttpService.get<{ value: IpcaMonthIpeaDto[] }>(this.jsonUrl).then(r => r.data?.value),
             3,
-            err => this.logger.log(`Retry Error: ${err}`)
+            err => this.logger.warn(`Retry Error: ${err}`)
         );
 
         return data;

api/src/main.ts

@@ -69,7 +69,7 @@ async function bootstrap() {
   };
 
   if (process.env.NODE_ENV === 'prod')
-    swaggerExpressOpts.customfavIcon = '/logo/logo.svg';
+    swaggerExpressOpts.customfavIcon = '/assets/logo/logo.svg';
 
   const document = SwaggerModule.createDocument(app, swaggerOpts);
   SwaggerModule.setup('/api', app, document, swaggerExpressOpts);

api/src/scheduler/services/scheduler.service.ts

@@ -1,4 +1,4 @@
-import { dateTimeToIsoStr, promiseParallelAll } from '@/@utils';
+import { dateTimeToIsoStr, promiseParallel } from '@/@utils';
 import { DataSource } from '@/core/enums/DataSource';
 import { BaseAssetService } from '@/core/services/BaseAssetService';
 import { GovBondDayTransparenteService } from '@/gov-bond/services/gov-bond-day-transparente.service';
@@ -31,10 +31,10 @@ export class SchedulerService {
             const funcs = Object.entries(this.services)
                 .filter(job => services.includes(job[0] as DataSource))
                 .map(job => () => this.runService(job[1]));
-            await promiseParallelAll(funcs, 2);
+            await promiseParallel(funcs, 2);
             this.logger.log(`Finished jobs (${funcs.length})`);
         } catch (err) {
-            this.logger.error(err.toString(), err.stack);
+            this.logger.error('Jobs failed');
         }
     }
 
@@ -59,6 +59,7 @@ export class SchedulerService {
             await service.getData({ minDate: new Date(0), maxDate: new Date(0) });
         } catch (err) {
             this.logger.error(`[${service.type}] ${err.toString()}`, err.stack);
+            throw err;
         }
     }
 }

api/src/search/search.module.ts

@@ -1,8 +1,12 @@
 import { SearchController } from '@/search/controllers/search.controller';
 import { SearchService } from '@/search/services/search.service';
+import { StockModule } from '@/stock/stock.module';
 import { Module } from '@nestjs/common';
 
 @Module({
+    imports: [
+        StockModule,
+    ],
     controllers: [
         SearchController,
     ],

api/src/search/services/search.service.ts

@@ -78,7 +78,10 @@ export class SearchService {
         },
     ];
 
-    constructor(private moduleRef: ModuleRef) {}
+    constructor(
+        private moduleRef: ModuleRef,
+        private stockYahooService: StockYahooService,
+    ) {}
 
     async getAssets(assetCodes: string, minDate: Date, maxDate: Date): Promise<AssetHistData<AssetData>[]> {
         minDate.setHours(0, 0, 0, 0);
@@ -88,6 +91,8 @@ export class SearchService {
         if (assets.length > 10) throw new Error('Too many assets (max = 10)');
         const assetsByType = groupBy(assets, a => a.rule.name);
 
+        const currencyRequests = new Map<string, Promise<AssetData[]>>();
+
         const tasks: (() => Promise<AssetHistData<AssetData>>)[] = Object.values(assetsByType).flatMap((assetRule) => {
             const rule = assetRule[0].rule;
 
@@ -112,8 +117,14 @@ export class SearchService {
                 if (currency != null) {
                     const assetCurrency = data.data[0]?.currency;
                     if (assetCurrency != null && assetCurrency !== currency) {
-                        const forexService: StockYahooService = await this.moduleRef.resolve(StockYahooService, undefined, { strict: false });
-                        const currencyData = await forexService.getData({ assetCode: `${assetCurrency}${currency}=X`, minDate, maxDate }).then(e => e.data);
+                        const currencyPair = `${assetCurrency}${currency}=X`;
+
+                        if (!currencyRequests.has(currencyPair)) {
+                            const currencyData = this.stockYahooService.getData({ assetCode: currencyPair, minDate, maxDate }).then(e => e.data);
+                            currencyRequests.set(currencyPair, currencyData);
+                        }
+
+                        const currencyData = await currencyRequests.get(currencyPair);
                         data.data = convertCurrency(data.data, currencyData);
                         data.data.forEach(e => e.assetCode = `${code}:${currency}`);
                     } else {