diff --git a/api/Dockerfile b/api/Dockerfile index 3e9e200..6fd098e 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -19,6 +19,7 @@ RUN npm run build FROM node:20.12.2-bullseye AS run ARG ENV=dev +ENV ENV=${ENV} ENV NODE_ENV=${ENV} ARG NODE_OPTIONS ENV NODE_OPTIONS=${NODE_OPTIONS} diff --git a/api/src/@utils/async/promise.ts b/api/src/@utils/async/promise.ts index 78a6564..cd26979 100644 --- a/api/src/@utils/async/promise.ts +++ b/api/src/@utils/async/promise.ts @@ -94,17 +94,26 @@ export async function promiseRetry(func: () => Promise, maxRetries: number } } -function isPromise(value: T | Promise): value is Promise { +export type TypedFunction = (...args: any[]) => any | Promise; +export type TypedFunctionWrapper = ((...args: Parameters) => ReturnType); +export type TypedSyncFunction = (...args: any[]) => T; +export type TypedAsyncFunction = (...args: any[]) => Promise; + +export function isPromise(value: T | Promise): value is Promise { return value instanceof Promise; } +export function isAsyncFunction(func: TypedFunction): func is TypedAsyncFunction { + return func.constructor.name == 'AsyncFunction'; +} + // Receives a value that could be already resolved or yet a Promise, // and calls the callback with the resolved value, // returning the resolved result or a new promise // export function runOrResolve(value: T, func: (resolved: T) => U): U; // export function runOrResolve(value: Promise, func: (resolved: T) => U): Promise; -export function runOrResolve(value: T | Promise, callback: (resolved: T) => U): U | Promise { +export function runOrResolve(value: T | Promise, callback: (resolved: T) => U, errCallback?: (rejected: any) => any): U | Promise { return isPromise(value) - ? value.then(resolved => callback(resolved as T)) + ? value.then(callback).catch(err => errCallback?.(err) ?? (err => { throw err; })(err)) : callback(value as T); } diff --git a/api/src/@utils/cache/memoize.ts b/api/src/@utils/cache/memoize.ts index 7b5719b..58db5e6 100644 --- a/api/src/@utils/cache/memoize.ts +++ b/api/src/@utils/cache/memoize.ts @@ -1,17 +1,14 @@ -import { jsonDateReviver, readFileSync, runOrResolve, tryParseJson, tryStringifyJson, writeFileSync } from '@/@utils'; +import { jsonDateReviver, readFileSync, runOrResolve, tryParseJson, tryStringifyJson, TypedFunction, TypedFunctionWrapper, writeFileSync } from '@/@utils'; import { createHash } from 'crypto'; import * as fs from 'fs'; import { isEqual, merge } from 'lodash'; -import { RFC_2822 } from 'moment-timezone'; import { tmpdir } from 'os'; import * as path from 'path'; // TYPES -export type TypedFunction = (...args: any[]) => any; - export type MemoizedProps = { _memoizeCache: MemoizeCache }; -export type Memoized = ((...args: Parameters) => ReturnType) & MemoizedProps; +export type Memoized = TypedFunctionWrapper & MemoizedProps; export interface MemoizeConfig { funcKey?: (config: MemoizeConfig) => string; @@ -38,25 +35,25 @@ export function isMemoizeCacheType(value: any): value is MemoizeCacheType { return Object.values(MemoizeCacheType).includes(value as MemoizeCacheType); } -export interface MemoizeCacheItem { +export interface MemoizeCacheItem { date: Date, - value: ReturnType, + value: ReturnType, } -export type MemoizeCacheMap = Record>; +export type MemoizeCacheMap = Record>; -export interface MemoizeCache { +export interface MemoizeCache { disabled?: boolean, config?: MemoizeConfig['cacheConfig'], cache: C, state: any, - memoizedFunc?: Memoized>, + memoizedFunc?: Memoized>, init: () => void, - get: (key: string) => MemoizeCacheItem | undefined, - set: (key: string, value: MemoizeCacheItem | Promise>) => void, + get: (key: string) => MemoizeCacheItem | undefined, + set: (key: string, value: MemoizeCacheItem | Promise>) => void, delete: (key: string) => void, flush: () => void, - invalidate: (predicate: (key: string, value: MemoizeCacheItem) => boolean) => void, + invalidate: (predicate: (key: string, value: MemoizeCacheItem) => boolean) => void, } // CONSTS @@ -68,8 +65,8 @@ export const globalMemoizeConfig: Partial = { // HOW-TO: globalMe } }; -const memoizeMemory = (config: MemoizeConfig) => { - const memoryCache: MemoizeCache> = { +const memoizeMemory = (config: MemoizeConfig) => { + const memoryCache: MemoizeCache> = { config: config.cacheConfig, cache: {}, state: {}, @@ -94,7 +91,7 @@ const memoizeMemory = (config: MemoizeConfig) => { return memoryCache; }; -const memoizeStorage = (config: MemoizeConfig) => { +const memoizeStorage = (config: MemoizeConfig) => { // TODO: do not load all to memory const getFileInfo = (cachePath: string) => { @@ -109,14 +106,14 @@ const memoizeStorage = (config: MemoizeConfig) => { }; // Create + Fetch + Modify + Save - const syncCache = (callback?: (cacheObj: MemoizeCacheMap) => MemoizeCacheMap | Promise>) => { + const syncCache = (callback?: (cacheObj: MemoizeCacheMap) => MemoizeCacheMap | Promise>) => { const fileInfoKey = '_fileInfo'; const fileInfo = getFileInfo(storageCache.config.cachePath); const oldFileInfo = storageCache.state[fileInfoKey]; const fileChanged = fileInfo && (oldFileInfo == null || oldFileInfo.hash !== fileInfo.hash); const prevCache = fileChanged - ? tryParseJson>(readFileSync(storageCache.config.cachePath), jsonDateReviver) + ? tryParseJson>(readFileSync(storageCache.config.cachePath), jsonDateReviver) : storageCache.cache; const newCache = callback?.(prevCache) ?? prevCache; @@ -127,18 +124,18 @@ const memoizeStorage = (config: MemoizeConfig) => { const newFileInfo = getFileInfo(storageCache.config.cachePath); storageCache.state[fileInfoKey] = newFileInfo; return storageCache.cache; - }); + }, err => err); } if (config.cacheConfig.cacheDir == null && config.cacheConfig.cachePath == null) throw new Error('Invalid cacheDir (null)'); const defaultConfig: MemoizeCacheConfig = { cachePath: path.join(config.cacheConfig.cacheDir, `memoize-${config.funcKey(config)}.json`) }; - const storageCache: MemoizeCache> = { + const storageCache: MemoizeCache> = { config: merge(defaultConfig, config.cacheConfig), cache: {}, state: {}, init: () => { - syncCache(); + syncCache(); }, get: key => { const cache = syncCache() as object; @@ -175,14 +172,14 @@ const cacheTypes: Record MemoizeCac // FUNCTIONS -export function memoize Promise>(func: Memoized, config: MemoizeConfig): Memoized { +export function memoize(func: Memoized, config: MemoizeConfig): Memoized { config = merge(globalMemoizeConfig, config); if (config.funcKey == null) throw new Error('Invalid funcKey'); const cache = isMemoizeCacheType(config.cacheType) ? cacheTypes[config.cacheType](config) : config.cacheType; - const memoized: Memoized = function (...args: Parameters): ReturnType { + const memoized: Memoized = function (...args: Parameters): ReturnType { if (cache.disabled) { config.onCall?.(config, args, cache); return func(...args); @@ -195,8 +192,8 @@ export function memoize Promise>(func: Memoiz const cacheEntry = cache.get(itemKey); if (cacheEntry !== undefined) { - const cacheAge = (Date.now() - cacheEntry.date.getTime()) / 1000; - const validEntry = config.cacheConfig.maxAge ? cacheAge < config.cacheConfig.maxAge : true; + const cacheAge = cacheEntry.date != null ? (Date.now() - cacheEntry.date.getTime()) / 1000 : null; + const validEntry = cacheAge != null && (config.cacheConfig.maxAge ? cacheAge < config.cacheConfig.maxAge : true); if (validEntry) return cacheEntry.value; else cache.delete(itemKey); } @@ -208,7 +205,7 @@ export function memoize Promise>(func: Memoiz return result; }; - memoized._memoizeCache = cache; // HOW-TO: (this.myFunc as Memoized)._memoizeCache.flush() + memoized._memoizeCache = cache; cache.memoizedFunc = memoized; return memoized; @@ -228,7 +225,7 @@ export function Memoize(config?: MemoizeConfig) { // use with "@Memoize()" config._instance = this; config.funcKey ??= () => `${config._instance?.constructor?.name}:${config._method.value?.name}`; - const memoized = memoize(config._method.value.bind(this), config); + const memoized = memoize(config._method.value.bind(config._instance), config).bind(config._instance); Object.defineProperty(this, name, { value: memoized, diff --git a/api/src/core/services/BaseAssetSgsService.ts b/api/src/core/services/BaseAssetSgsService.ts index d6ddd7d..33e0a83 100644 --- a/api/src/core/services/BaseAssetSgsService.ts +++ b/api/src/core/services/BaseAssetSgsService.ts @@ -70,7 +70,7 @@ export abstract class BaseAssetSgsService extends BaseAsset let data = await promiseRetry( () => HttpService.get(this.jsonUrl, { responseType: 'text' }).then(r => r.data), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); data = tryParseJson(data, undefined, false); diff --git a/api/src/gov-bond/services/gov-bond-day-last-td.service.ts b/api/src/gov-bond/services/gov-bond-day-last-td.service.ts index a22f52a..198f3c3 100644 --- a/api/src/gov-bond/services/gov-bond-day-last-td.service.ts +++ b/api/src/gov-bond/services/gov-bond-day-last-td.service.ts @@ -113,7 +113,7 @@ export class GovBondDayLastTdService extends BaseAssetService { const data = await promiseRetry(() => HttpService.get(this.jsonUrl, { httpsAgent }).then(r => r.data), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); return data; diff --git a/api/src/gov-bond/services/gov-bond-day-sisweb.service.ts b/api/src/gov-bond/services/gov-bond-day-sisweb.service.ts index 0ac39ee..2b2435f 100644 --- a/api/src/gov-bond/services/gov-bond-day-sisweb.service.ts +++ b/api/src/gov-bond/services/gov-bond-day-sisweb.service.ts @@ -139,7 +139,7 @@ export class GovBondDaySiswebService extends BaseAssetService { const file = await promiseRetry( () => HttpService.get(e.asset.url!, { responseType: 'arraybuffer' }).then(r => Buffer.from(r.data, 'binary')), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); assetsData.push({ @@ -178,13 +178,13 @@ export class GovBondDaySiswebService extends BaseAssetService { validateStatus: status => status >= 200 && status < 303, }).then(r => r.headers["set-cookie"]), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); html = await promiseRetry( () => HttpService.get(this.indexUrl, { headers: { 'Cookie': cookie[0] } }).then(r => r.data), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); const htmlMain = html.match(/
[\s\S]*?<\/div>/)?.[0]; diff --git a/api/src/gov-bond/services/gov-bond-day-transparente.service.ts b/api/src/gov-bond/services/gov-bond-day-transparente.service.ts index e7a1ce9..a1f699d 100644 --- a/api/src/gov-bond/services/gov-bond-day-transparente.service.ts +++ b/api/src/gov-bond/services/gov-bond-day-transparente.service.ts @@ -128,7 +128,7 @@ export class GovBondDayTransparenteService extends BaseAssetService { const file = await promiseRetry( () => HttpService.get(this.csvUrl, { responseType: 'arraybuffer' }).then(r => Buffer.from(r.data, 'binary')), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); const assetsData: GovBondDayTransparenteDto = {}; diff --git a/api/src/ipca/services/ipca-month-ipea.service.ts b/api/src/ipca/services/ipca-month-ipea.service.ts index 03a545d..02aa1fb 100644 --- a/api/src/ipca/services/ipca-month-ipea.service.ts +++ b/api/src/ipca/services/ipca-month-ipea.service.ts @@ -68,7 +68,7 @@ export class IpcaMonthIpeaService extends BaseAssetService { const data = await promiseRetry( () => HttpService.get<{ value: IpcaMonthIpeaDto[] }>(this.jsonUrl).then(r => r.data?.value), 3, - err => this.logger.log(`Retry Error: ${err}`) + err => this.logger.warn(`Retry Error: ${err}`) ); return data; diff --git a/api/src/scheduler/services/scheduler.service.ts b/api/src/scheduler/services/scheduler.service.ts index 372a76f..5d5c4df 100644 --- a/api/src/scheduler/services/scheduler.service.ts +++ b/api/src/scheduler/services/scheduler.service.ts @@ -1,4 +1,4 @@ -import { dateTimeToIsoStr, promiseParallelAll } from '@/@utils'; +import { dateTimeToIsoStr, promiseParallel } from '@/@utils'; import { DataSource } from '@/core/enums/DataSource'; import { BaseAssetService } from '@/core/services/BaseAssetService'; import { GovBondDayTransparenteService } from '@/gov-bond/services/gov-bond-day-transparente.service'; @@ -31,10 +31,10 @@ export class SchedulerService { const funcs = Object.entries(this.services) .filter(job => services.includes(job[0] as DataSource)) .map(job => () => this.runService(job[1])); - await promiseParallelAll(funcs, 2); + await promiseParallel(funcs, 2); this.logger.log(`Finished jobs (${funcs.length})`); } catch (err) { - this.logger.error(err.toString(), err.stack); + this.logger.error('Jobs failed'); } } @@ -59,6 +59,7 @@ export class SchedulerService { await service.getData({ minDate: new Date(0), maxDate: new Date(0) }); } catch (err) { this.logger.error(`[${service.type}] ${err.toString()}`, err.stack); + throw err; } } } \ No newline at end of file diff --git a/api/src/search/services/search.service.ts b/api/src/search/services/search.service.ts index 1791b58..e81719c 100644 --- a/api/src/search/services/search.service.ts +++ b/api/src/search/services/search.service.ts @@ -88,6 +88,8 @@ export class SearchService { if (assets.length > 10) throw new Error('Too many assets (max = 10)'); const assetsByType = groupBy(assets, a => a.rule.name); + const currencyRequests = new Map>(); + const tasks: (() => Promise>)[] = Object.values(assetsByType).flatMap((assetRule) => { const rule = assetRule[0].rule; @@ -112,8 +114,15 @@ export class SearchService { if (currency != null) { const assetCurrency = data.data[0]?.currency; if (assetCurrency != null && assetCurrency !== currency) { - const forexService: StockYahooService = await this.moduleRef.resolve(StockYahooService, undefined, { strict: false }); - const currencyData = await forexService.getData({ assetCode: `${assetCurrency}${currency}=X`, minDate, maxDate }).then(e => e.data); + const currencyPair = `${assetCurrency}${currency}=X`; + + if (!currencyRequests.has(currencyPair)) { + const forexService: StockYahooService = await this.moduleRef.resolve(StockYahooService, undefined, { strict: false }); + const currencyData = forexService.getData({ assetCode: currencyPair, minDate, maxDate }).then(e => e.data); + currencyRequests.set(currencyPair, currencyData); + } + + const currencyData = await currencyRequests.get(currencyPair); data.data = convertCurrency(data.data, currencyData); data.data.forEach(e => e.assetCode = `${code}:${currency}`); } else {