From d0ff49a573fc46c02f958331384ef3f89b3367e2 Mon Sep 17 00:00:00 2001 From: slayer1551 Date: Wed, 1 Jun 2022 13:21:28 +0100 Subject: [PATCH 1/7] Typo --- public/app/js/confirms.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/app/js/confirms.js b/public/app/js/confirms.js index 3b732315..52f38a36 100644 --- a/public/app/js/confirms.js +++ b/public/app/js/confirms.js @@ -1,8 +1,8 @@ const popupmessage = Vue.component("popup-message", { props: ["job", "deletec", "requeuec", "createc"], template: ` -
Job Deleted successfull
-
Job Requeue successfull
-
Job Created successfull
+
Job Deleted successful
+
Job Requeue successful
+
Job Created successful
`, }); From b936d3b3c660cf4093f5c4e44acf08e86393b3bb Mon Sep 17 00:00:00 2001 From: slayer1551 Date: Wed, 1 Jun 2022 13:29:15 +0100 Subject: [PATCH 2/7] Fix new job window modal from not closing --- public/app/js/newJob.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/app/js/newJob.js b/public/app/js/newJob.js index ac13e063..3c925fc3 100644 --- a/public/app/js/newJob.js +++ b/public/app/js/newJob.js @@ -38,7 +38,6 @@ const newJob = Vue.component("new-job", { .then((data) => { this.$emit("popup-message"); this.$emit("refresh-data"); - this.$refs.Close.click(); this.clear(); }) .catch(console.log); @@ -79,7 +78,7 @@ const newJob = Vue.component("new-job", {
- +
From 9e14bd6675ac0019026b7b0b3b24a850678dcded Mon Sep 17 00:00:00 2001 From: slayer1551 Date: Wed, 1 Jun 2022 13:32:27 +0100 Subject: [PATCH 3/7] AWS Document DB Support $facet is not available in AWS Document DB and throws an error. Created an error check that if it fails with the standard query to try the AWS query to create the pages and filters. --- lib/controllers/agendash.js | 166 +++++++++++++++++++++++------------- 1 file changed, 108 insertions(+), 58 deletions(-) diff --git a/lib/controllers/agendash.js b/lib/controllers/agendash.js index e78b9280..821f3fcb 100644 --- a/lib/controllers/agendash.js +++ b/lib/controllers/agendash.js @@ -39,7 +39,7 @@ module.exports = function (agenda, options) { if (job) { preMatch.name = job; } - + if (options.query && options.property) { if (options.isObjectId) { preMatch[options.property] = ObjectId(options.query); @@ -48,6 +48,7 @@ module.exports = function (agenda, options) { } else { preMatch[options.property] = { $regex: options.query, $options: "i" }; } + } const postMatch = {}; @@ -56,69 +57,118 @@ module.exports = function (agenda, options) { } const collection = agenda._collection.collection || agenda._collection; - return collection - .aggregate([ - { $match: preMatch }, - { - $sort: { - nextRunAt: -1, - lastRunAt: -1, - lastFinishedAt: -1, - }, + const aggregateQuery = [ + { $match: preMatch }, + { + $sort: { + nextRunAt: -1, + lastRunAt: -1, + lastFinishedAt: -1, }, - { - $project: { - job: "$$ROOT", - _id: "$$ROOT._id", - running: { - $and: ["$lastRunAt", { $gt: ["$lastRunAt", "$lastFinishedAt"] }], - }, - scheduled: { - $and: ["$nextRunAt", { $gte: ["$nextRunAt", new Date()] }], - }, - queued: { - $and: [ - "$nextRunAt", - { $gte: [new Date(), "$nextRunAt"] }, - { $gte: ["$nextRunAt", "$lastFinishedAt"] }, - ], - }, - completed: { - $and: [ - "$lastFinishedAt", - { $gt: ["$lastFinishedAt", "$failedAt"] }, - ], - }, - failed: { - $and: [ - "$lastFinishedAt", - "$failedAt", - { $eq: ["$lastFinishedAt", "$failedAt"] }, - ], - }, - repeating: { - $and: ["$repeatInterval", { $ne: ["$repeatInterval", null] }], - }, + }, + { + $project: { + job: "$$ROOT", + _id: "$$ROOT._id", + running: { + $and: ["$lastRunAt", { $gt: ["$lastRunAt", "$lastFinishedAt"] }], + }, + scheduled: { + $and: ["$nextRunAt", { $gte: ["$nextRunAt", new Date()] }], + }, + queued: { + $and: [ + "$nextRunAt", + { $gte: [new Date(), "$nextRunAt"] }, + { $gte: ["$nextRunAt", "$lastFinishedAt"] }, + ], + }, + completed: { + $and: [ + "$lastFinishedAt", + { $gt: ["$lastFinishedAt", "$failedAt"] }, + ], + }, + failed: { + $and: [ + "$lastFinishedAt", + "$failedAt", + { $eq: ["$lastFinishedAt", "$failedAt"] }, + ], + }, + repeating: { + $and: ["$repeatInterval", { $ne: ["$repeatInterval", null] }], }, }, - { $match: postMatch }, - { - $facet: { - pages: [ - { $count: "totalMatchs" }, - { - $project: { - totalPages: { - $ceil: { $divide: ["$totalMatchs", options.limit] }, - }, + }, + { $match: postMatch } + + + ]; + const standardMongoDBQuery = [ + { + $facet: { + pages: [ + { $count: "totalMatchs" }, + { + $project: { + totalPages: { + $ceil: { $divide: ["$totalMatchs", options.limit] }, }, }, - ], - filtered: [{ $skip: options.skip }, { $limit: options.limit }], - }, + }, + ], + filtered: [{ $skip: options.skip }, { $limit: options.limit }], }, - ]) - .toArray(); + }, + { + $project: { + total:12 + } + } + ] + const awsDocumentDBQuery = [ + { + $group: { + _id: null, + totalMatchs: { + $sum: 1 + }, + results: { + $push: '$$ROOT' + } + } + }, + { + $project: { + pages: [{totalPages: '$totalMatchs'}], + filtered: { + $slice: [ + '$results', + options.skip, + options.limit + ] + } + } + } + ] + return collection + .aggregate([...aggregateQuery,...standardMongoDBQuery]) + .toArray() + .then((result) => { + return result; + }) + .catch((error) => { + return collection.aggregate([...aggregateQuery,...awsDocumentDBQuery]) + .toArray() + .then((result) => { + return result; + }) + .catch((error) => { + return error; + }) + }); + }; const getOverview = async () => { From d90ea5f2f0240edd1905f87b1caac401cb97960f Mon Sep 17 00:00:00 2001 From: slayer1551 Date: Wed, 1 Jun 2022 13:58:59 +0100 Subject: [PATCH 4/7] Fix totalPage count Fixed the total page count --- lib/controllers/agendash.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/controllers/agendash.js b/lib/controllers/agendash.js index 821f3fcb..1e2a833e 100644 --- a/lib/controllers/agendash.js +++ b/lib/controllers/agendash.js @@ -106,6 +106,7 @@ module.exports = function (agenda, options) { ]; const standardMongoDBQuery = [ + { $facet: { pages: [ @@ -162,11 +163,14 @@ module.exports = function (agenda, options) { return collection.aggregate([...aggregateQuery,...awsDocumentDBQuery]) .toArray() .then((result) => { + result[0].pages[0].totalPages = Math.ceil(result[0].pages[0].totalPages / options.limit); return result; }) .catch((error) => { return error; }) + + }); }; From 32b3687df518437b771a195b73619fc4a79037dc Mon Sep 17 00:00:00 2001 From: slayer1551 Date: Wed, 1 Jun 2022 14:38:54 +0100 Subject: [PATCH 5/7] Update agendash.js removed line uses for testing --- lib/controllers/agendash.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/controllers/agendash.js b/lib/controllers/agendash.js index 1e2a833e..96e2afc1 100644 --- a/lib/controllers/agendash.js +++ b/lib/controllers/agendash.js @@ -120,11 +120,6 @@ module.exports = function (agenda, options) { }, ], filtered: [{ $skip: options.skip }, { $limit: options.limit }], - }, - }, - { - $project: { - total:12 } } ] From a76b9e5c229dbf546af7c0d16623f0cd7a941321 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 15 Sep 2022 18:39:23 +0300 Subject: [PATCH 6/7] Temp: Remove the attempt to do a $facet query Make a proper patch where it is configurable later. --- lib/controllers/agendash.js | 54 +++++++++---------------------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/lib/controllers/agendash.js b/lib/controllers/agendash.js index 96e2afc1..ce944a99 100644 --- a/lib/controllers/agendash.js +++ b/lib/controllers/agendash.js @@ -39,7 +39,7 @@ module.exports = function (agenda, options) { if (job) { preMatch.name = job; } - + if (options.query && options.property) { if (options.isObjectId) { preMatch[options.property] = ObjectId(options.query); @@ -48,7 +48,7 @@ module.exports = function (agenda, options) { } else { preMatch[options.property] = { $regex: options.query, $options: "i" }; } - + } const postMatch = {}; @@ -102,27 +102,8 @@ module.exports = function (agenda, options) { }, }, { $match: postMatch } - - ]; - const standardMongoDBQuery = [ - { - $facet: { - pages: [ - { $count: "totalMatchs" }, - { - $project: { - totalPages: { - $ceil: { $divide: ["$totalMatchs", options.limit] }, - }, - }, - }, - ], - filtered: [{ $skip: options.skip }, { $limit: options.limit }], - } - } - ] const awsDocumentDBQuery = [ { $group: { @@ -148,26 +129,17 @@ module.exports = function (agenda, options) { } } ] - return collection - .aggregate([...aggregateQuery,...standardMongoDBQuery]) - .toArray() - .then((result) => { - return result; - }) - .catch((error) => { - return collection.aggregate([...aggregateQuery,...awsDocumentDBQuery]) - .toArray() - .then((result) => { - result[0].pages[0].totalPages = Math.ceil(result[0].pages[0].totalPages / options.limit); - return result; - }) - .catch((error) => { - return error; - }) - - - }); - + return collection.aggregate([...aggregateQuery,...awsDocumentDBQuery]) + .toArray() + .then((result) => { + result[0].pages[0].totalPages = Math.ceil(result[0].pages[0].totalPages / options.limit); + return result; + }) + .catch((error) => { + console.error('docdb query error', error); + console.debug(JSON.stringify([...aggregateQuery,...awsDocumentDBQuery])); + return error; + }) }; const getOverview = async () => { From 53e2739fdc05200978d675f945c755f12712d590 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Thu, 15 Sep 2022 18:40:06 +0300 Subject: [PATCH 7/7] Temp: Apply a 1000 document limit to the jobs query The DocumentDB query barfs on large data sets. This is a temporary fix until the documentdb query is properly fixed. --- lib/controllers/agendash.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/controllers/agendash.js b/lib/controllers/agendash.js index ce944a99..6bc51255 100644 --- a/lib/controllers/agendash.js +++ b/lib/controllers/agendash.js @@ -105,6 +105,11 @@ module.exports = function (agenda, options) { ]; const awsDocumentDBQuery = [ + // Limit to 1000 documents because this crashes on larger data sets + // FIXME fix properly + { + $limit: 1000 + }, { $group: { _id: null,