Add event timestamp to repetitions

.dockerignore

@@ -2,12 +2,10 @@
 node_modules
 coverage
 /**/tests
-migrations
 tools
 bin
 
 # config files
-migrate-mongo-config.js
 jest-mongodb-config.js
 jest.config.js
 ecosystem.config.js

lib/utils.js

@@ -26,13 +26,14 @@ module.exports.deepMerge = deepMerge;
  *
  * @param {object|Array} source - source object
  * @param {object|Array} target - target object
+ * @param {Array<string>} [requiredFields] - fields to leave in diff object
  * @returns {object}
  */
-function deepDiff(source, target) {
+function deepDiff(source, target, requiredFields= []) {
   if (typeOf(target) === 'array') {
     return arrayDiff(source, target);
   } else if (typeOf(target) === 'object') {
-    return objectDiff(source, target);
+    return objectDiff(source, target, requiredFields);
   } else if (source !== target) {
     return target;
   } else {
@@ -62,10 +63,11 @@ function arrayDiff(source, target) {
  *
  * @param {object} objectA - first object for comparing
  * @param {object} objectB - second object for comparing
+ * @param {Array<string>} requiredFields - fields to leave
  *
  * @returns {object}
  */
-function objectDiff(objectA, objectB) {
+function objectDiff(objectA, objectB, requiredFields = []) {
   const diffObject = {};
 
   /**
@@ -93,6 +95,10 @@ function objectDiff(objectA, objectB) {
     }
 
     if (objectAItem === objectBItem) {
+      if (requiredFields.includes(prop)) {
+        diffObject[prop] = objectAItem;
+      }
+
       return;
     }
 

lib/utils.test.js

@@ -135,4 +135,12 @@ describe('Utils', () => {
       expect(merge).toEqual(testCase.expectedMerge);
     });
   });
+
+  test('should leave required fields', () => {
+    const data = dataProvider[1];
+
+    const diff = utils.deepDiff(data.sourceObject, data.targetObject, ['a']);
+
+    expect(diff.a).toEqual(data.sourceObject.a);
+  })
 });

migrations/20210826165556-add-timestamp-to-repetitions.js

@@ -0,0 +1,50 @@
+/**
+ * This migration sets timestamp for event repetitions if it was omitted because it's the same as original event one
+ */
+module.exports = {
+  async up(db, client) {
+    const collections = await db.listCollections({}, {
+      authorizedCollections: true,
+      nameOnly: true,
+    }).toArray();
+
+    const projectIds = [];
+    const REPETITIONS = 'repetitions';
+    const EVENTS = 'events';
+
+    collections.forEach((collection) => {
+      if (/repetitions/.test(collection.name)) {
+        projectIds.push(collection.name.split(':')[1]);
+      }
+    });
+
+    const session = client.startSession();
+
+    try {
+      session.withTransaction(async () => {
+        for (const projectId of projectIds) {
+          const originalEvents = await db.collection(`${EVENTS}:${projectId}`).find({}).toArray();
+          const repetitions = await db.collection(`${REPETITIONS}:${projectId}`).find({
+            'payload.timestamp': {$eq: null}
+          }).toArray();
+
+          for (const event of originalEvents) {
+            const eventRepetitions = repetitions.filter(rep => rep.groupHash === event.groupHash);
+
+            for (const repetition of eventRepetitions) {
+              await db.collection(`${REPETITIONS}:${projectId}`).updateOne({
+                _id: repetition._id,
+              }, {
+                $set: {
+                  'payload.timestamp': event.payload.timestamp,
+                },
+              });
+            }
+          }
+        }
+      })
+    } finally {
+      session.endSession();
+    }
+  },
+};

package.json

@@ -44,7 +44,7 @@
         "run-limiter": "yarn worker hawk-worker-limiter"
     },
     "dependencies": {
-        "@hawk.so/nodejs": "^2.1.0",
+        "@hawk.so/nodejs": "^3.0.10",
         "@types/amqplib": "^0.5.13",
         "@types/jest": "^25.2.1",
         "@types/mongodb": "^3.5.15",

workers/grouper/src/index.ts

@@ -146,8 +146,10 @@ export default class GrouperWorker extends Worker {
 
       /**
        * Save event's repetitions
+       *
+       * Leave timestamp in diff for database queries
        */
-      const diff = utils.deepDiff(existedEvent.payload, task.event);
+      const diff = utils.deepDiff(existedEvent.payload, task.event, [ 'timestamp' ]);
 
       const newRepetition = {
         groupHash: uniqueEventHash,

workers/javascript/src/index.ts

@@ -119,7 +119,7 @@ export default class JavascriptEventWorker extends EventWorker {
                */
               HawkCatcher.send(error, {
                 payload: event.payload,
-              });
+              } as any);
 
               return event.payload.backtrace[index];
             });

workers/limiter/src/index.ts

@@ -327,7 +327,7 @@ export default class LimiterWorker extends Worker {
 
       HawkCatcher.send(error, {
         workspaceId: workspace._id,
-      });
+      } as any);
 
       throw error;
     }

workers/paymaster/src/index.ts

@@ -167,7 +167,7 @@ export default class PaymasterWorker extends Worker {
 
           HawkCatcher.send(error, {
             workspaceId: workspace._id,
-          });
+          } as any);
 
           return false;
         }

yarn.lock

@@ -279,13 +279,22 @@
     minimatch "^3.0.4"
     strip-json-comments "^3.1.1"
 
-"@hawk.so/nodejs@^2.1.0":
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/@hawk.so/nodejs/-/nodejs-2.2.0.tgz#d72f028b2b39eaf6ac1a8b5f711ab9faa7649cb3"
+"@hawk.so/nodejs@^3.0.10":
+  version "3.0.10"
+  resolved "https://registry.yarnpkg.com/@hawk.so/nodejs/-/nodejs-3.0.10.tgz#11e34f2572f53088913b1d45e3abfaa391386904"
+  integrity sha512-KpVk34n6ttX21KEJlGmKi7Pg4bh+x3x1Ot8O1Xy4ZRb+ob208PU/ThDEOeJgf/QH3QBTTNnNWK/HvjSW81TkHg==
   dependencies:
-    axios "^0.19.2"
+    "@hawk.so/types" "^0.1.15"
+    axios "^0.21.1"
     stack-trace "^0.0.10"
 
+"@hawk.so/types@^0.1.15":
+  version "0.1.15"
+  resolved "https://registry.yarnpkg.com/@hawk.so/types/-/types-0.1.15.tgz#9a561dda47fab91f3093e32cbdfaf83b1eb000c1"
+  integrity sha512-LHZdjmyIbAQ+0Og9Px34BTYF5YM0fto9bSluONmgVVr6Ctu2nPqPibNF1s9PEEisrtRxzuUQX8ea/OAmOJhK3w==
+  dependencies:
+    "@types/mongodb" "^3.5.34"
+
 "@istanbuljs/load-nyc-config@^1.0.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced"
@@ -1147,12 +1156,6 @@ aws4@^1.8.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
 
-axios@^0.19.2:
-  version "0.19.2"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27"
-  dependencies:
-    follow-redirects "1.5.10"
-
 axios@^0.20.0:
   version "0.20.0"
   resolved "https://registry.yarnpkg.com/axios/-/axios-0.20.0.tgz#057ba30f04884694993a8cd07fa394cff11c50bd"
@@ -1960,12 +1963,6 @@ [email protected]:
   dependencies:
     ms "^2.1.1"
 
-debug@=3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261"
-  dependencies:
-    ms "2.0.0"
-
 debug@^2.2.0, debug@^2.3.3, debug@^2.6.9, debug@~2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -2714,12 +2711,6 @@ [email protected]:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
 
-[email protected]:
-  version "1.5.10"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a"
-  dependencies:
-    debug "=3.1.0"
-
 follow-redirects@^1.10.0:
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.0.tgz#b42e8d93a2a7eea5ed88633676d6597bc8e384db"