From 185860b0fe534830eb41e6b1f4c307157cadd190 Mon Sep 17 00:00:00 2001 From: ayasayadi1 Date: Thu, 7 Sep 2023 14:55:47 +0000 Subject: [PATCH 1/7] WIP: script --- bin/scripts/fix-data/recheck_merges.js | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100755 bin/scripts/fix-data/recheck_merges.js diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js new file mode 100755 index 00000000000..e2133b3f93d --- /dev/null +++ b/bin/scripts/fix-data/recheck_merges.js @@ -0,0 +1,89 @@ +/** + * Description: This script is used to recheck merges and update drill data with new uid + * Server: countly + * Path: $(countly dir)/bin/scripts/fix-data + * Command: node recheck_merges.js + */ + + +var asyncjs = require("async"); +const pluginManager = require('../../../plugins/pluginManager.js'); +const dataviews = require('../../../plugins/drill/api/parts/data/dataviews.js'); + +Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) { + console.log("Connected to databases."); + //get all apps + countlyDb.collection("apps").find().toArray(function(err, apps) { + if (err) { + return close(err); + } + //get all drill collections + drillDb.collections(function(err, collections) { + if (err) { + return close(err); + } + //for each app + asyncjs.eachOf(apps, function(app, index, done) { + var app_id = app._id; + if (err) { + return done(); + } + //get all users with merges + countlyDb.collection('app_users' + app_id).find({merges: {$gt: 0}}).toArray(function(err, users) { + if (err) { + return done(); + } + //for each user + asyncjs.eachOf(users, function(user, index, done) { + var new_uid = user.uid; + //get all merges to this user + countlyDb.collection('app_user_merges' + app_id).find({merged_to: new_uid}).toArray(function(err, merges) { + if (err) { + return done(); + } + //for each merge, check if old uid still exists in any drill collection + asyncjs.eachOf(merges, function(merge, index, done) { + let old_uid = merge._id; + asyncjs.eachOf(collections, function(collection, index, done) { + collection = collection.collectionName; + //get event with old_uid + drillDb.collection(collection).find({uid: old_uid}, {_id: 1}).limit(1).toArray(function(err, events) { + if (err) { + return done(); + } + //if there is an event with the old uid, update them to the new uid + if (events[0]) { + console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); + drillDb.collection(drillDb.getCollectionName(events[0], app_id)).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}, function() { + dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app_id, function() { + return done(); + }); + }); + } + }); + }, function() { + done(); + }); + }, function() { + done(); + }); + }); + }, function() { + done(); + }); + }); + }, function(err) { + close(err); + }); + }); + }); + + function close(err) { + if (err) { + console.log("Error: ", err); + } + countlyDb.close(); + drillDb.close(); + console.log("Done."); + } +}); \ No newline at end of file From 31c3c43a9f38712ef3f1b33a13cdd6c88cfe7f3e Mon Sep 17 00:00:00 2001 From: ayasayadi1 Date: Fri, 8 Sep 2023 14:55:11 +0000 Subject: [PATCH 2/7] fixes --- bin/scripts/fix-data/recheck_merges.js | 85 +++++++++++--------------- 1 file changed, 35 insertions(+), 50 deletions(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index e2133b3f93d..495f892bf4d 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -13,64 +13,28 @@ const dataviews = require('../../../plugins/drill/api/parts/data/dataviews.js'); Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) { console.log("Connected to databases."); //get all apps - countlyDb.collection("apps").find().toArray(function(err, apps) { - if (err) { + countlyDb.collection("apps").find({}, {_id: 1, name: 1}).toArray(function(err, apps) { + if (err || !apps || !apps.length) { return close(err); } //get all drill collections drillDb.collections(function(err, collections) { - if (err) { + if (err || !collections || !collections.length) { return close(err); } //for each app - asyncjs.eachOf(apps, function(app, index, done) { - var app_id = app._id; - if (err) { - return done(); - } - //get all users with merges - countlyDb.collection('app_users' + app_id).find({merges: {$gt: 0}}).toArray(function(err, users) { - if (err) { - return done(); + asyncjs.eachSeries(apps, function(app, done) { + //get users with merges + const usersCursor = countlyDb.collection('app_users' + app._id).find({merges: {$gt: 0}}, {_id: 1, uid: 1, merged_uid: 1}); + //for each user + usersCursor.next(function(err, user) { + if (err || !user) { + return done(err); + } + //check if old uid still exists in drill collections + if (user.merged_uid) { + processUser(user.merged_uid, user.uid, collections, app); } - //for each user - asyncjs.eachOf(users, function(user, index, done) { - var new_uid = user.uid; - //get all merges to this user - countlyDb.collection('app_user_merges' + app_id).find({merged_to: new_uid}).toArray(function(err, merges) { - if (err) { - return done(); - } - //for each merge, check if old uid still exists in any drill collection - asyncjs.eachOf(merges, function(merge, index, done) { - let old_uid = merge._id; - asyncjs.eachOf(collections, function(collection, index, done) { - collection = collection.collectionName; - //get event with old_uid - drillDb.collection(collection).find({uid: old_uid}, {_id: 1}).limit(1).toArray(function(err, events) { - if (err) { - return done(); - } - //if there is an event with the old uid, update them to the new uid - if (events[0]) { - console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); - drillDb.collection(drillDb.getCollectionName(events[0], app_id)).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}, function() { - dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app_id, function() { - return done(); - }); - }); - } - }); - }, function() { - done(); - }); - }, function() { - done(); - }); - }); - }, function() { - done(); - }); }); }, function(err) { close(err); @@ -78,6 +42,27 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" }); }); + function processUser(old_uid, new_uid, collections, app) { + asyncjs.eachSeries(collections, function(collection, done) { + collection = collection.collectionName; + drillDb.collection(collection).find({uid: old_uid}, {_id: 1}).limit(1).toArray().then(async function(err, events) { + if (err || !events || !events.length) { + done(); + } + if (events && events[0]) { + console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); + drillDb.collection(drillDb.getCollectionName(events[0], app._id)).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}, function() { + dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app._id, function() { + return done(); + }); + }); + } + }); + }, function(err) { + return close(err); + }); + } + function close(err) { if (err) { console.log("Error: ", err); From 345a11cee9f486410a687d915f311b37701ab2e2 Mon Sep 17 00:00:00 2001 From: ayasayadi1 Date: Mon, 11 Sep 2023 15:08:49 +0000 Subject: [PATCH 3/7] bug fixes --- bin/scripts/fix-data/recheck_merges.js | 91 ++++++++++++++++---------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index 495f892bf4d..5715393e553 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -6,61 +6,82 @@ */ -var asyncjs = require("async"); +const asyncjs = require("async"); const pluginManager = require('../../../plugins/pluginManager.js'); const dataviews = require('../../../plugins/drill/api/parts/data/dataviews.js'); +const common = require("../../../api/utils/common.js"); Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) { console.log("Connected to databases."); + common.db = countlyDb; + common.drillDb = drillDb; //get all apps - countlyDb.collection("apps").find({}, {_id: 1, name: 1}).toArray(function(err, apps) { - if (err || !apps || !apps.length) { - return close(err); + try { + const apps = await countlyDb.collection("apps").find({}, {_id: 1, name: 1}).toArray(); + if (!apps || !apps.length) { + return close(); } - //get all drill collections - drillDb.collections(function(err, collections) { - if (err || !collections || !collections.length) { - return close(err); + try { + //get all drill collections + const collections = await drillDb.collections(); + if (!collections || !collections.length) { + return close(); } - //for each app - asyncjs.eachSeries(apps, function(app, done) { + //for each app serially process users + asyncjs.eachSeries(apps, async function(app) { //get users with merges const usersCursor = countlyDb.collection('app_users' + app._id).find({merges: {$gt: 0}}, {_id: 1, uid: 1, merged_uid: 1}); //for each user - usersCursor.next(function(err, user) { - if (err || !user) { - return done(err); - } + while (await usersCursor.hasNext()) { + const user = await usersCursor.next(); //check if old uid still exists in drill collections - if (user.merged_uid) { - processUser(user.merged_uid, user.uid, collections, app); + if (user && user.merged_uid) { + await processUser(user.merged_uid, user.uid, collections, app); } - }); + } }, function(err) { - close(err); + return close(err); }); - }); - }); + } + catch (err) { + return close(err); + } + } + catch (err) { + return close(err); + } - function processUser(old_uid, new_uid, collections, app) { - asyncjs.eachSeries(collections, function(collection, done) { - collection = collection.collectionName; - drillDb.collection(collection).find({uid: old_uid}, {_id: 1}).limit(1).toArray().then(async function(err, events) { - if (err || !events || !events.length) { - done(); + async function processUser(old_uid, new_uid, collections, app) { + console.log("Processing user ", new_uid, "for app ", app.name); + for (let i = 0; i < collections.length; i++) { + const collection = collections[i].collectionName; + try { + const events = await drillDb.collection(collection).find({uid: old_uid}, {_id: 1}).limit(1).toArray(); + if (!events || !events.length) { + continue; } if (events && events[0]) { console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); - drillDb.collection(drillDb.getCollectionName(events[0], app._id)).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}, function() { - dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app._id, function() { - return done(); - }); - }); + try { + await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}); + if (dataviews) { + try { + await dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app._id, function() {}); + } + catch (err) { + console.log("Error updating dataviews for app ", app.name, "with old uid ", old_uid, "to new uid ", new_uid, "error: ", err); + } + } + } + catch (err) { + console.log("Error updating collection ", collection, "for app ", app.name, "with old uid ", old_uid, "to new uid ", new_uid, "error: ", err); + } } - }); - }, function(err) { - return close(err); - }); + } + catch (err) { + console.log("Error finding events with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "error: ", err); + } + } } function close(err) { From f145c26c4c899d65a50bbf845f12a89088edb697 Mon Sep 17 00:00:00 2001 From: ayasayadi1 Date: Mon, 11 Sep 2023 15:48:14 +0000 Subject: [PATCH 4/7] more fixes --- bin/scripts/fix-data/recheck_merges.js | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index 5715393e553..b2888c7f2c7 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -53,6 +53,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" async function processUser(old_uid, new_uid, collections, app) { console.log("Processing user ", new_uid, "for app ", app.name); + var has_merges = false; for (let i = 0; i < collections.length; i++) { const collection = collections[i].collectionName; try { @@ -61,17 +62,10 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" continue; } if (events && events[0]) { + has_merges = true; console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); try { await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}); - if (dataviews) { - try { - await dataviews.mergeUserTimes({uid: old_uid}, {uid: new_uid}, app._id, function() {}); - } - catch (err) { - console.log("Error updating dataviews for app ", app.name, "with old uid ", old_uid, "to new uid ", new_uid, "error: ", err); - } - } } catch (err) { console.log("Error updating collection ", collection, "for app ", app.name, "with old uid ", old_uid, "to new uid ", new_uid, "error: ", err); @@ -82,6 +76,19 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" console.log("Error finding events with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "error: ", err); } } + if (has_merges && dataviews) { + await new Promise((resolve, reject) => { + dataviews.mergeUserTimes({ uid: old_uid }, { uid: new_uid }, app._id, function(err) { + if (err) { + reject(err); + } + else { + console.log("Updated user times for app ", app.name, "with old uid ", old_uid, "to new uid ", new_uid); + resolve(); + } + }); + }); + } } function close(err) { From 58af96f78f3b961d8abaeb01c117719991c7d8e4 Mon Sep 17 00:00:00 2001 From: Anna Sosina Date: Mon, 11 Sep 2023 21:28:11 +0300 Subject: [PATCH 5/7] Update recheck_merges.js --- bin/scripts/fix-data/recheck_merges.js | 34 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index b2888c7f2c7..6be3ebc4f88 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -10,6 +10,7 @@ const asyncjs = require("async"); const pluginManager = require('../../../plugins/pluginManager.js'); const dataviews = require('../../../plugins/drill/api/parts/data/dataviews.js'); const common = require("../../../api/utils/common.js"); +const drillCommon = require("../../../plugins/drill/api/common.js"); Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("countly_drill")]).then(async function([countlyDb, drillDb]) { console.log("Connected to databases."); @@ -22,14 +23,11 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" return close(); } try { - //get all drill collections - const collections = await drillDb.collections(); - if (!collections || !collections.length) { - return close(); - } //for each app serially process users asyncjs.eachSeries(apps, async function(app) { + console.log("Processing app ", app.name); //get users with merges + var collections = await getDrillCollections(app._id); //get all drill collections for this app const usersCursor = countlyDb.collection('app_users' + app._id).find({merges: {$gt: 0}}, {_id: 1, uid: 1, merged_uid: 1}); //for each user while (await usersCursor.hasNext()) { @@ -51,8 +49,28 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" return close(err); } + + async function getDrillCollections(app_id){ + var collections = []; + try { + var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)}); + var list = ["[CLY]_session","[CLY]_crash","[CLY]_view","[CLY]_action","[CLY]_push_action","[CLY]_star_rating","[CLY]_nps","[CLY]_survey","[CLY]_apm_network","[CLY]_apm_device"]; + + if (events && events.list) { + list = list.concat(events.list); + } + for (let i = 0; i < list.length; i++) { + var collectionName = drillCommon.getCollectionName(list[i], app_id); + collections.push({collectionName: collectionName}); + } + } + catch(err2){ + console.log("Error getting drill collections for app ", app_id, "error: ", err2); + } + return collections; + } async function processUser(old_uid, new_uid, collections, app) { - console.log("Processing user ", new_uid, "for app ", app.name); + console.log("Processing user ", old_uid," -> ",new_uid, "for app ", app.name); var has_merges = false; for (let i = 0; i < collections.length; i++) { const collection = collections[i].collectionName; @@ -76,7 +94,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" console.log("Error finding events with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "error: ", err); } } - if (has_merges && dataviews) { + if (dataviews) { await new Promise((resolve, reject) => { dataviews.mergeUserTimes({ uid: old_uid }, { uid: new_uid }, app._id, function(err) { if (err) { @@ -99,4 +117,4 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" drillDb.close(); console.log("Done."); } -}); \ No newline at end of file +}); From 2dfd111ec3b1285dba04473ef4a2f4ec3d41d5e3 Mon Sep 17 00:00:00 2001 From: ayasayadi1 Date: Mon, 11 Sep 2023 18:59:21 +0000 Subject: [PATCH 6/7] eslint fixes --- bin/scripts/fix-data/recheck_merges.js | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index 6be3ebc4f88..af7bb1bbe7a 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -49,13 +49,12 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" return close(err); } - - async function getDrillCollections(app_id){ + async function getDrillCollections(app_id) { var collections = []; try { - var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)}); - var list = ["[CLY]_session","[CLY]_crash","[CLY]_view","[CLY]_action","[CLY]_push_action","[CLY]_star_rating","[CLY]_nps","[CLY]_survey","[CLY]_apm_network","[CLY]_apm_device"]; - + var events = await countlyDb.collection("events").findOne({_id: common.db.ObjectID(app_id)}); + var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device"]; + if (events && events.list) { list = list.concat(events.list); } @@ -64,14 +63,14 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" collections.push({collectionName: collectionName}); } } - catch(err2){ - console.log("Error getting drill collections for app ", app_id, "error: ", err2); + catch (err) { + console.log("Error getting drill collections for app ", app_id, "error: ", err); } return collections; } + async function processUser(old_uid, new_uid, collections, app) { - console.log("Processing user ", old_uid," -> ",new_uid, "for app ", app.name); - var has_merges = false; + console.log("Processing user ", old_uid, " -> ", new_uid, "for app ", app.name); for (let i = 0; i < collections.length; i++) { const collection = collections[i].collectionName; try { @@ -80,7 +79,6 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" continue; } if (events && events[0]) { - has_merges = true; console.log("Found at least one event with old uid ", old_uid, "in collection ", collection, "for app ", app.name, "updating to new uid", new_uid); try { await drillDb.collection(collection).update({uid: old_uid}, {'$set': {uid: new_uid}}, {multi: true}); From a65f52c5c52d7f3565caa52ce51240383ce0605b Mon Sep 17 00:00:00 2001 From: Anna Sosina Date: Tue, 12 Sep 2023 10:28:02 +0300 Subject: [PATCH 7/7] Make sure there are no duplicates in events list --- bin/scripts/fix-data/recheck_merges.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/scripts/fix-data/recheck_merges.js b/bin/scripts/fix-data/recheck_merges.js index af7bb1bbe7a..6f0848160e9 100755 --- a/bin/scripts/fix-data/recheck_merges.js +++ b/bin/scripts/fix-data/recheck_merges.js @@ -56,7 +56,11 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection(" var list = ["[CLY]_session", "[CLY]_crash", "[CLY]_view", "[CLY]_action", "[CLY]_push_action", "[CLY]_star_rating", "[CLY]_nps", "[CLY]_survey", "[CLY]_apm_network", "[CLY]_apm_device"]; if (events && events.list) { - list = list.concat(events.list); + for (var p = 0; p < events.list.length; p++) { + if (list.indexOf(events.list[p]) === -1) { + list.push(events.list[p]); + } + } } for (let i = 0; i < list.length; i++) { var collectionName = drillCommon.getCollectionName(list[i], app_id);