diff --git a/package-lock.json b/package-lock.json index 79a3771..3734d11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "bull-arena": "^3.29.3", "bull-master": "^1.0.5", "bull-prom": "ejhayes/bull-prom#master", - "bullmq": "^1.86.2", + "bullmq": "^3.11.0", "commander": "^9.3.0", "envalid": "^7.3.1", "forever-monitor": "^3.0.3", @@ -4298,6 +4298,89 @@ "node": ">=10" } }, + "node_modules/bull-master/node_modules/bullmq": { + "version": "1.91.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.91.1.tgz", + "integrity": "sha512-u7dat9I8ZwouZ651AMZkBSvB6NVUPpnAjd4iokd9DM41whqIBnDjuL11h7+kEjcpiDKj6E+wxZiER00FqirZQg==", + "optional": true, + "dependencies": { + "cron-parser": "^4.6.0", + "get-port": "6.1.2", + "glob": "^8.0.3", + "ioredis": "^5.2.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "optional": true, + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "optional": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "optional": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "optional": true, + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/bull-master/node_modules/bullmq/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "optional": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bull-master/node_modules/bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -4654,14 +4737,13 @@ } }, "node_modules/bullmq": { - "version": "1.90.2", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.90.2.tgz", - "integrity": "sha512-GBCvEdMG2mnR0Jkgc/Bv6ZBRbswkRopHJsyJik65/VXVO2/2jb8HC0+iC0cALmn7aCKpJGoVwtUmH0FLw7yLYQ==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.11.0.tgz", + "integrity": "sha512-l9/Wy1rKGQGgnlkKM50q6jknzEf2v9lhIh4xva71TqopWORUuGBtFDrHAQn/X6m+fQnXWiG5FVL0fnHdHhcQeA==", "dependencies": { "cron-parser": "^4.6.0", - "get-port": "6.1.2", "glob": "^8.0.3", - "ioredis": "^5.2.2", + "ioredis": "^5.3.0", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "semver": "^7.3.7", @@ -4669,17 +4751,6 @@ "uuid": "^9.0.0" } }, - "node_modules/bullmq/node_modules/get-port": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", - "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/bullmq/node_modules/uuid": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", @@ -8330,14 +8401,14 @@ } }, "node_modules/ioredis": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", - "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -9878,9 +9949,9 @@ } }, "node_modules/luxon": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.0.3.tgz", - "integrity": "sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==", "engines": { "node": ">=12" } @@ -17696,6 +17767,69 @@ "uuid": "^8.3.0" } }, + "bullmq": { + "version": "1.91.1", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.91.1.tgz", + "integrity": "sha512-u7dat9I8ZwouZ651AMZkBSvB6NVUPpnAjd4iokd9DM41whqIBnDjuL11h7+kEjcpiDKj6E+wxZiER00FqirZQg==", + "optional": true, + "requires": { + "cron-parser": "^4.6.0", + "get-port": "6.1.2", + "glob": "^8.0.3", + "ioredis": "^5.2.2", + "lodash": "^4.17.21", + "msgpackr": "^1.6.2", + "semver": "^7.3.7", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + }, + "dependencies": { + "cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "optional": true, + "requires": { + "luxon": "^3.2.1" + } + }, + "denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "optional": true + }, + "get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "optional": true + }, + "ioredis": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", + "optional": true, + "requires": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + } + }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "optional": true + } + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -17967,14 +18101,13 @@ "requires": {} }, "bullmq": { - "version": "1.90.2", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-1.90.2.tgz", - "integrity": "sha512-GBCvEdMG2mnR0Jkgc/Bv6ZBRbswkRopHJsyJik65/VXVO2/2jb8HC0+iC0cALmn7aCKpJGoVwtUmH0FLw7yLYQ==", + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.11.0.tgz", + "integrity": "sha512-l9/Wy1rKGQGgnlkKM50q6jknzEf2v9lhIh4xva71TqopWORUuGBtFDrHAQn/X6m+fQnXWiG5FVL0fnHdHhcQeA==", "requires": { "cron-parser": "^4.6.0", - "get-port": "6.1.2", "glob": "^8.0.3", - "ioredis": "^5.2.2", + "ioredis": "^5.3.0", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "semver": "^7.3.7", @@ -17982,11 +18115,6 @@ "uuid": "^9.0.0" }, "dependencies": { - "get-port": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", - "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==" - }, "uuid": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", @@ -20825,14 +20953,14 @@ "requires": {} }, "ioredis": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.2.3.tgz", - "integrity": "sha512-gQNcMF23/NpvjCaa1b5YycUyQJ9rBNH2xP94LWinNpodMWVUPP5Ai/xXANn/SM7gfIvI62B5CCvZxhg5pOgyMw==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.3.2.tgz", + "integrity": "sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==", "requires": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", - "denque": "^2.0.1", + "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", @@ -22003,9 +22131,9 @@ } }, "luxon": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.0.3.tgz", - "integrity": "sha512-+EfHWnF+UT7GgTnq5zXg3ldnTKL2zdv7QJgsU5bjjpbH17E3qi/puMhQyJVYuCq+FRkogvB5WB6iVvUr+E4a7w==" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==" }, "macos-release": { "version": "2.5.0", diff --git a/package.json b/package.json index d247192..f85439b 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "bull-arena": "^3.29.3", "bull-master": "^1.0.5", "bull-prom": "ejhayes/bull-prom#master", - "bullmq": "^1.86.2", + "bullmq": "^3.11.0", "commander": "^9.3.0", "envalid": "^7.3.1", "forever-monitor": "^3.0.3", diff --git a/src/bull/bull-queues.service.ts b/src/bull/bull-queues.service.ts index f06ffd8..e65580a 100644 --- a/src/bull/bull-queues.service.ts +++ b/src/bull/bull-queues.service.ts @@ -2,7 +2,7 @@ import { ConfigService } from '@app/config/config.service'; import { InjectLogger, LoggerService } from '@app/logger'; import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Mutex, withTimeout } from 'async-mutex'; -import { Queue, QueueScheduler } from 'bullmq'; +import { Queue } from 'bullmq'; import { RedisService } from 'nestjs-redis'; import { TypedEmitter } from 'tiny-typed-emitter2'; import { @@ -39,7 +39,6 @@ const REDIS_CONFIG_NOTIFY_KEYSPACE_EVENTS_FLAGS = 'A$K'; export class BullQueuesService implements OnModuleInit, OnModuleDestroy { private _initialized = false; private readonly _queues: { [queueName: string]: Queue } = {}; - private readonly _schedulers: { [queueName: string]: QueueScheduler } = {}; private readonly _redisMutex = withTimeout(new Mutex(), 10000); private readonly _bullMutex = withTimeout(new Mutex(), 10000); @@ -117,30 +116,6 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { this._queues[queueKey].on('ioredis:close', () => { this.removeQueue(queuePrefix, queueName); }); - /** - * From: https://docs.bullmq.io/guide/connections - * - * Every class will consume at least one Redis connection, but it - * is also possible to reuse connections in some situations. For example, - * the Queue and Worker classes can accept an existing ioredis instance, and - * by that reusing that connection, however QueueScheduler and QueueEvents - * cannot do that because they require blocking connections to Redis, which - * makes it impossible to reuse them. - */ - this._schedulers[queueKey] = new QueueScheduler(queueName, { - prefix: queuePrefix, - connection: { - host: this.configService.config.REDIS_HOST, - port: this.configService.config.REDIS_PORT, - password: this.configService.config.REDIS_PASSWORD, - family: this.configService.config.REDIS_FAMILY, - }, - }); - this._schedulers[queueKey].on('error', (err) => { - Error.captureStackTrace(err); - this.logger.error(err.stack); - this.removeQueue(queuePrefix, queueName); - }); this.eventEmitter.emit( EVENT_TYPES.QUEUE_CREATED, new QueueCreatedEvent(queuePrefix, this._queues[queueKey]), @@ -158,14 +133,12 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { try { await this._queues[queueKey].close(); - await this._schedulers[queueKey].close(); } catch (err) { // in the event of an error just ignore it and move on this.logger.error(err); } delete this._queues[queueKey]; - delete this._schedulers[queueKey]; this.eventEmitter.emit( EVENT_TYPES.QUEUE_REMOVED, @@ -470,10 +443,7 @@ export class BullQueuesService implements OnModuleInit, OnModuleDestroy { this.eventEmitter.removeAllListeners(); // close all connections - for (const queue of [ - Object.values(this._queues), - Object.values(this._schedulers), - ].flat()) { + for (const queue of [Object.values(this._queues)].flat()) { this.logger.warn(`Closing queue: ${queue.name}`); await new Promise(async (resolve) => { (await queue.client).on('close', () => {