From 863bbafab279459f6e3004d2aaf67551f3a63df0 Mon Sep 17 00:00:00 2001 From: Eugen Klymniuk Date: Fri, 26 Jul 2024 07:34:43 +0100 Subject: [PATCH] feat(csi-334): updated proxyCache to use redis cluster (#536) * feat(csi-334): updated proxyCache to use redis cluster * feat(csi-334): updated proxyCache to use redis cluster * feat(csi-334): updated proxyCache to use redis cluster * feat(csi-334): updated proxyCache to use redis cluster * chore(snapshot): 14.1.0-snapshot.0 * chore(snapshot): 14.1.0-snapshot.1 * chore(snapshot): 14.1.0-snapshot.2 * chore(snapshot): 14.1.0-snapshot.3 * chore(snapshot): 14.1.0-snapshot.4 * chore(snapshot): 14.1.0-snapshot.5 * chore(snapshot): 14.1.0-snapshot.6 * chore(snapshot): 14.1.0-snapshot.7 * chore(snapshot): 14.1.0-snapshot.8 * chore(snapshot): 14.1.0-snapshot.9 * chore(snapshot): 14.1.0-snapshot.10 * chore(snapshot): 14.1.0-snapshot.11 * chore(snapshot): 14.1.0-snapshot.12 * chore(snapshot): 14.1.0-snapshot.13 * chore(snapshot): 14.1.0-snapshot.14 * chore(snapshot): 14.1.0-snapshot.15 * chore(snapshot): 14.1.0-snapshot.16 * chore(snapshot): 14.1.0-snapshot.17 --- docker-compose.yml | 39 +++-- docker/ml-api-adapter/default.json | 5 +- package-lock.json | 138 ++++++++++++++---- package.json | 8 +- scripts/_wait4_all.js | 4 +- test/fixtures/index.js | 12 +- .../handlers/notification/index.test.js | 19 ++- 7 files changed, 165 insertions(+), 60 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e8e5c695..e6e15206 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,14 @@ networks: ml-mojaloop-net: name: ml-mojaloop-net +x-redis-node: &REDIS_NODE + image: docker.io/bitnami/redis-cluster:6.2.14 + environment: &REDIS_ENVS + ALLOW_EMPTY_PASSWORD: yes + REDIS_NODES: redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5 + networks: + - ml-mojaloop-net + services: ml-api-adapter: image: mojaloop/ml-api-adapter:local @@ -175,15 +183,28 @@ services: timeout: 20s retries: 10 interval: 30s - redis: - image: redis:7.2.5 - container_name: redis - networks: - - ml-mojaloop-net + + redis-node-0: + <<: *REDIS_NODE environment: - - ALLOW_EMPTY_PASSWORD=yes - - REDIS_PORT=6379 - - REDIS_REPLICATION_MODE=master - - REDIS_TLS_ENABLED=no + <<: *REDIS_ENVS + REDIS_CLUSTER_CREATOR: yes + depends_on: + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + - redis-node-5 ports: - "6379:6379" + + redis-node-1: + <<: *REDIS_NODE + redis-node-2: + <<: *REDIS_NODE + redis-node-3: + <<: *REDIS_NODE + redis-node-4: + <<: *REDIS_NODE + redis-node-5: + <<: *REDIS_NODE diff --git a/docker/ml-api-adapter/default.json b/docker/ml-api-adapter/default.json index 96d67176..18e6c6bd 100644 --- a/docker/ml-api-adapter/default.json +++ b/docker/ml-api-adapter/default.json @@ -3,8 +3,9 @@ "enabled": true, "type": "redis", "proxyConfig": { - "host": "redis", - "port": 6379 + "cluster": [ + { "host": "redis-node-0", "port": 6379 } + ] } }, "HUB_PARTICIPANT": { diff --git a/package-lock.json b/package-lock.json index 44821f52..3ce62176 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mojaloop/ml-api-adapter", - "version": "14.1.0-proxy.16", + "version": "14.1.0-snapshot.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mojaloop/ml-api-adapter", - "version": "14.1.0-proxy.16", + "version": "14.1.0-snapshot.17", "license": "Apache-2.0", "dependencies": { "@hapi/basic": "7.0.2", @@ -19,9 +19,9 @@ "@hapi/vision": "7.0.3", "@mojaloop/central-services-error-handling": "13.0.1", "@mojaloop/central-services-health": "15.0.0", - "@mojaloop/central-services-logger": "11.3.1", + "@mojaloop/central-services-logger": "11.5.0", "@mojaloop/central-services-metrics": "12.0.8", - "@mojaloop/central-services-shared": "18.6.3", + "@mojaloop/central-services-shared": "18.7.0-snapshot.0", "@mojaloop/central-services-stream": "11.3.1", "@mojaloop/event-sdk": "14.1.1", "@mojaloop/sdk-standard-components": "18.3.0", @@ -40,7 +40,7 @@ }, "devDependencies": { "@mojaloop/database-lib": "11.0.6", - "@mojaloop/inter-scheme-proxy-cache-lib": "^1.4.0", + "@mojaloop/inter-scheme-proxy-cache-lib": "2.0.0-snapshot.1", "@types/hapi__joi": "^17.1.14", "audit-ci": "^7.1.0", "get-port": "5.1.1", @@ -1679,15 +1679,43 @@ } }, "node_modules/@mojaloop/central-services-logger": { - "version": "11.3.1", - "resolved": "https://registry.npmjs.org/@mojaloop/central-services-logger/-/central-services-logger-11.3.1.tgz", - "integrity": "sha512-XVU2K5grE1ZcIyxUXeMlvoVkeIcs9y1/0EKxa2Bk5sEbqXUtHuR8jqbAGlwaUIi9T9YWZRJyVC77nOQe/X1teA==", + "version": "11.5.0", + "resolved": "https://registry.npmjs.org/@mojaloop/central-services-logger/-/central-services-logger-11.5.0.tgz", + "integrity": "sha512-pH73RiJ5fKTBTSdLocp1vPBad1D+Kh0HufdcfjLaBQj3dIBq72si0k+Z3L1MeOmMqMzpj+8M/he/izlgqJjVJA==", "dependencies": { - "@types/node": "^20.12.7", "parse-strings-in-object": "2.0.0", "rc": "1.2.8", "safe-stable-stringify": "^2.4.3", - "winston": "3.13.0" + "winston": "3.13.1" + } + }, + "node_modules/@mojaloop/central-services-logger/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@mojaloop/central-services-logger/node_modules/winston": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.1.tgz", + "integrity": "sha512-SvZit7VFNvXRzbqGHsv5KSmgbEYR5EiQfDAL9gxYkRqa934Hnk++zze0wANKtMHcy/gI4W/3xmSDwlhf865WGw==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.6.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" } }, "node_modules/@mojaloop/central-services-metrics": { @@ -1699,19 +1727,20 @@ } }, "node_modules/@mojaloop/central-services-shared": { - "version": "18.6.3", - "resolved": "https://registry.npmjs.org/@mojaloop/central-services-shared/-/central-services-shared-18.6.3.tgz", - "integrity": "sha512-GTMNxBB4lhjrW7V52OmZvuWKKx7IywmyihAfmcmSJ1zCtb+yL1CzF/pM4slOx2d6taE9Pn+q3S2Ucf/ZV2QzuA==", + "version": "18.7.0-snapshot.0", + "resolved": "https://registry.npmjs.org/@mojaloop/central-services-shared/-/central-services-shared-18.7.0-snapshot.0.tgz", + "integrity": "sha512-4oyVSOUbbU73DZa0A5gK09mxaK4CBvQ3BVUMRPBpzzL4T+pcfuZZJr5Uj1fW4A/Z2XWpHh0/O2y3u+Jd0szAaA==", "dependencies": { "@hapi/catbox": "12.1.1", "@hapi/catbox-memory": "5.0.1", - "@mojaloop/inter-scheme-proxy-cache-lib": "1.4.0", + "@mojaloop/inter-scheme-proxy-cache-lib": "2.0.0-snapshot.1", "axios": "1.7.2", "clone": "2.1.2", "dotenv": "16.4.5", "env-var": "7.5.0", "event-stream": "4.0.1", - "immutable": "4.3.6", + "fast-safe-stringify": "^2.1.1", + "immutable": "4.3.7", "lodash": "4.17.21", "mustache": "4.2.0", "openapi-backend": "5.10.6", @@ -1720,7 +1749,7 @@ "shins": "2.6.0", "uuid4": "2.0.3", "widdershins": "^4.0.1", - "yaml": "2.4.5" + "yaml": "2.5.0" }, "peerDependencies": { "@mojaloop/central-services-error-handling": ">=13.x.x", @@ -1846,12 +1875,12 @@ "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/@mojaloop/inter-scheme-proxy-cache-lib": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@mojaloop/inter-scheme-proxy-cache-lib/-/inter-scheme-proxy-cache-lib-1.4.0.tgz", - "integrity": "sha512-jmAWWdjZxjxlSQ+wt8aUcMYOneVo1GNbIIs7yK/R2K9DBtKb0aYle2mWwdjm9ovk6zSWL2a9lH+n3hq7kb08Wg==", + "version": "2.0.0-snapshot.1", + "resolved": "https://registry.npmjs.org/@mojaloop/inter-scheme-proxy-cache-lib/-/inter-scheme-proxy-cache-lib-2.0.0-snapshot.1.tgz", + "integrity": "sha512-3cIs1U0Ufk16YFw9eR4QrrkO+hSHLUkoTAoxN5wM09/HIfE9+upZCnn0R+roB52FWCOe4TgURFMMIKo0E6HPuw==", "dependencies": { - "@mojaloop/central-services-logger": "^11.3.1", - "ajv": "^8.16.0", + "@mojaloop/central-services-logger": "11.4.5", + "ajv": "^8.17.1", "convict": "^6.2.4", "fast-safe-stringify": "^2.1.1", "ioredis": "^5.4.1" @@ -1860,6 +1889,46 @@ "node": ">=18.x" } }, + "node_modules/@mojaloop/inter-scheme-proxy-cache-lib/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@mojaloop/inter-scheme-proxy-cache-lib/node_modules/@mojaloop/central-services-logger": { + "version": "11.4.5", + "resolved": "https://registry.npmjs.org/@mojaloop/central-services-logger/-/central-services-logger-11.4.5.tgz", + "integrity": "sha512-nCKEIinB/Zx3routZhcGd+//IKd9oThpGggTde4rNLJ6O4nVJgHSW6pZIzd1T+Mj34yBhOPidhBa0piLBKcZtQ==", + "dependencies": { + "parse-strings-in-object": "2.0.0", + "rc": "1.2.8", + "safe-stable-stringify": "^2.4.3", + "winston": "3.13.1" + } + }, + "node_modules/@mojaloop/inter-scheme-proxy-cache-lib/node_modules/winston": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.1.tgz", + "integrity": "sha512-SvZit7VFNvXRzbqGHsv5KSmgbEYR5EiQfDAL9gxYkRqa934Hnk++zze0wANKtMHcy/gI4W/3xmSDwlhf865WGw==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.6.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/@mojaloop/sdk-standard-components": { "version": "18.3.0", "resolved": "https://registry.npmjs.org/@mojaloop/sdk-standard-components/-/sdk-standard-components-18.3.0.tgz", @@ -2661,14 +2730,14 @@ } }, "node_modules/ajv": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.16.0.tgz", - "integrity": "sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dependencies": { "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -6526,6 +6595,11 @@ "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "node_modules/fast-uri": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.1.tgz", + "integrity": "sha512-MWipKbbYiYI0UC7cl8m/i/IWTqfC8YXsqjzybjddLsFjStroQzsHXkc73JutMvBiXmOvapk+axIl79ig5t55Bw==" + }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -8258,9 +8332,9 @@ } }, "node_modules/immutable": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", - "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==" + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.7.tgz", + "integrity": "sha512-1hqclzwYwjRDFLjcFxOM5AYkkG0rpFPpr1RLPMEuGczoS7YA8gLhy8SWXYRAA/XwfEHpfo3cw5JGioS32fnMRw==" }, "node_modules/import-fresh": { "version": "3.3.0", @@ -18508,9 +18582,9 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yaml": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.5.tgz", - "integrity": "sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.0.tgz", + "integrity": "sha512-2wWLbGbYDiSqqIKoPjar3MPgB94ErzCtrNE1FdqGuaO0pi2JGjmE8aW8TDZwzU7vuxcGRdL/4gPQwQ7hD5AMSw==", "bin": { "yaml": "bin.mjs" }, diff --git a/package.json b/package.json index de005ab4..51efcd2f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@mojaloop/ml-api-adapter", - "version": "14.1.0-proxy.16", + "version": "14.1.0-snapshot.17", "description": "Convert from ML API to/from internal Central Services messaging format", "license": "Apache-2.0", "private": true, @@ -89,9 +89,9 @@ "@hapi/vision": "7.0.3", "@mojaloop/central-services-error-handling": "13.0.1", "@mojaloop/central-services-health": "15.0.0", - "@mojaloop/central-services-logger": "11.3.1", + "@mojaloop/central-services-logger": "11.5.0", "@mojaloop/central-services-metrics": "12.0.8", - "@mojaloop/central-services-shared": "18.6.3", + "@mojaloop/central-services-shared": "18.7.0-snapshot.0", "@mojaloop/central-services-stream": "11.3.1", "@mojaloop/event-sdk": "14.1.1", "@mojaloop/sdk-standard-components": "18.3.0", @@ -110,7 +110,7 @@ }, "devDependencies": { "@mojaloop/database-lib": "11.0.6", - "@mojaloop/inter-scheme-proxy-cache-lib": "^1.4.0", + "@mojaloop/inter-scheme-proxy-cache-lib": "2.0.0-snapshot.1", "@types/hapi__joi": "^17.1.14", "audit-ci": "^7.1.0", "get-port": "5.1.1", diff --git a/scripts/_wait4_all.js b/scripts/_wait4_all.js index cd34ea29..fbcd0a41 100755 --- a/scripts/_wait4_all.js +++ b/scripts/_wait4_all.js @@ -16,8 +16,8 @@ const expectedContainers = [ 'ml_mysql' ] -let retries = 40 -const waitTimeMs = 60000 +let retries = 50 +const waitTimeMs = 20000 async function main () { const waitingMap = {} diff --git a/test/fixtures/index.js b/test/fixtures/index.js index 7972f0a4..9e00479c 100644 --- a/test/fixtures/index.js +++ b/test/fixtures/index.js @@ -170,6 +170,15 @@ const createMessageProtocol = (eventType = 'prepare', eventAction = 'prepare', p } } +const proxyCacheConfigDto = ({ host = 'localhost' } = {}) => ({ + type: 'redis', + proxyConfig: { + cluster: [ + { host, port: 6379 } + ] + } +}) + module.exports = { buildTransfer, buildFXTransfer, @@ -179,5 +188,6 @@ module.exports = { generateTransferId, generateParentTestSpan, createMessageProtocol, - createProducerConfig + createProducerConfig, + proxyCacheConfigDto } diff --git a/test/integration/handlers/notification/index.test.js b/test/integration/handlers/notification/index.test.js index 673e571d..e37d456f 100644 --- a/test/integration/handlers/notification/index.test.js +++ b/test/integration/handlers/notification/index.test.js @@ -29,18 +29,19 @@ const Test = require('tapes')(require('tape')) const Uuid = require('uuid4') const db = require('@mojaloop/database-lib').Db -const Config = require('../../../../src/lib/config') -const centralLedgerConfig = require('../../../../docker/central-ledger/default.json') +const Logger = require('@mojaloop/central-services-logger') +const proxyLib = require('@mojaloop/inter-scheme-proxy-cache-lib') const { Kafka: KafkaUtil, HeaderValidation, Request } = require('@mojaloop/central-services-shared').Util const Enum = require('@mojaloop/central-services-shared').Enum const encodePayload = require('@mojaloop/central-services-shared').Util.StreamingProtocol.encodePayload const Kafka = require('@mojaloop/central-services-stream').Util -const { Action } = Enum.Events.Event -const Fixtures = require('../../../fixtures/index') + +const Config = require('../../../../src/lib/config') +const centralLedgerConfig = require('../../../../docker/central-ledger/default.json') const { prepare } = require('../../../../src/domain/transfer/index') -const Logger = require('@mojaloop/central-services-logger') -const proxyLib = require('@mojaloop/inter-scheme-proxy-cache-lib') +const Fixtures = require('../../../fixtures') +const { Action } = Enum.Events.Event const EventTypes = Enum.Events.Event.Type const EventActions = Enum.Events.Event.Action const GeneralTopicTemplate = Config.KAFKA_CONFIG.TOPIC_TEMPLATES.GENERAL_TOPIC_TEMPLATE.TEMPLATE @@ -73,10 +74,8 @@ Test('Notification Handler', notificationHandlerTest => { notificationHandlerTest.test('should', async notificationTest => { let proxy notificationTest.test('connect proxy lib', async test => { - proxy = proxyLib.createProxyCache('redis', { - host: 'localhost', - port: 6379 - }) + const { type, proxyConfig } = Fixtures.proxyCacheConfigDto() + proxy = proxyLib.createProxyCache(type, proxyConfig) await proxy.connect() test.pass('Connected proxy lib') test.end()