Merge branch 'next' into SER-787-widgets

Countly · Sep 7, 2023 · 4dc1457 · 4dc1457
2 parents f871df0 + adc0236
commit 4dc1457
Show file tree

Hide file tree

Showing 255 changed files with 7,171 additions and 1,285 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -251,16 +251,4 @@ jobs:
         run: |
           /sbin/my_init &
           npm install
-          node bin/scripts/test.connection.js
-  notify:
-    runs-on: ubuntu-latest
-    needs: [ install, lint, test-api-core, test-api-plugins ]
-    steps:
-      - name: Send slack notification
-        id: slack
-        if: always()
-        uses: slackapi/[email protected]
-        with:
-          payload: "{\"url\":\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\",\"status\":\"${{ job.status }}\"}"
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          node bin/scripts/test.connection.js
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,12 @@
 api/node_modules/
 .idea
 .vscode
+.cache
+.npm
+.npmrc
+.local
+bin/config/nginx/
+bin/config/supervisord.conf
 *_en.properties
 frontend/express/node_modules/
 api/config.js
@@ -18,6 +24,7 @@ frontend/express/public/sdk/
 **/.DS_Store
 **/node_modules
 bin/scripts/nghttp2
+frontend/express/public/userimages/
 frontend/express/public/images/*/
 !frontend/express/public/images/leaflet/
 !frontend/express/public/images/annotation/
@@ -32,4 +39,7 @@ frontend/express/public/images/*/
 frontend/express/public/stylesheets/styles/manifest.css
 frontend/express/public/stylesheets/vue/clyvue.css
 frontend/express/public/stylesheets/styles/manifest.css.map
-frontend/express/public/stylesheets/vue/clyvue.css.map
+frontend/express/public/stylesheets/vue/clyvue.css.map
+log/
+log/supervisord/
+plugins/plugins.json.*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,66 @@
+## Version 23.06.9
+Fixes:
+- [push] Adding push/notifications endpoint, deprecating push/user endpoint
+- [push] Docs: fixing required fields, adding example
+- [core] If user property is redacted, it should not show list of values in Data manager too
+- [hooks] Fix for firebase url validation
+
+Enterprise fixes:
+- [push] Allowing push token to be filtered from cohorts
+- [drill] Fix table download bug
+- [data-manager] Transformation - Validation for transformation failing on incoming request
+
+## Version 23.06.8
+Fixes:
+- [core] Automatic license management installation script
+- [core] Fixes for meta merging script
+- [core] Adding automation tests to base code
+- [core] Improvements for views to correctly count unique user count for one view if multiple views with the same name are sent in a single request.
+- [crashes] Fix query builder option source
+- [dbviewer] Fixed bug with  the filter being ignored on export if any value is null in the filter.
+
+Enterprise fixes:
+- [cohorts] Fixed pipeline for the top metric calculation to return only top 3 values, not all possible.
+- [cohorts] Fixes for data fetching upon loading cohorts table
+- [data-manager] Fixed a typo in the import schema function
+- [data-manager] Fixes to prevent Duplicate Events due to Transform Merge
+- [drill] Drill meta document conversion to type string on overflow based on document size
+- [funnels] Funnel edit form does not decode funnel name on open
+- [push] Fixing drill query by push token
+- [push] User Profiles querying by push tokens & messages
+- [push] User details rendering of push tokens present in a profile
+
+
+## Version 23.06.7
+Fixes:
+- [consent] Fixes for user consent table
+- [core] Do not track region and location if tracking city data is diabled
+- [core] Fixed bug with selecting app containing special characters in dbviewer
+- [core] Fixed encoding for application list
+- [core] Fixed issue with session analytics showing incorrect returning users count for single day.
+- [crashes] Fix query builder property types
+- [db-viewer] Fixed issues with export if used query contains objectID
+- [db-viewer] Fixed using correct filename in export
+- [graph-notes] Fixed issue with editing graph note
+- [report-manager] Fixed export table issue
+
+Enterprise fixes:
+- [cohorts] Revalidate user property rule like '7days' or '30days' in nightly job.
+
+## Version 23.06.6
+Fixes:
+- [crashes] Limit custom properties key length in crashgroup
+- [core] Fix issue with encoding and decoding for \u0000
+- [core] Decode carriers name in carriers table
+- [apps] display old salt property value too
+- [db-viewer] Add search to DB Viewer apps filter and do alphabetical order
+- [dashboard] Fix error in charts added in time-series type in User Analytics widgets
+- [dashboard] Fix dashboard widgets showing no data incorrectly
+
+Enterprise fixes:
+- [users][drill] Correctly Match events for user timeline in user profiles
+- [data-manager] Fix issue with incorrect period in regeneration of aggregate data
+
 ## Version 23.06.5
 Fixes:
 - [core] Fixes for pluginManager callPromisedAppMethod

diff --git a/Gruntfile.js b/Gruntfile.js
@@ -195,7 +195,9 @@ module.exports = function(grunt) {
                     'frontend/express/public/core/geo-countries/javascripts/countly.widgets.geo.js',
                     'frontend/express/public/core/home/javascripts/countly.models.js',
                     'frontend/express/public/core/home/javascripts/countly.views.js',
-                    'frontend/express/public/core/notes/javascripts/countly.views.js'
+                    'frontend/express/public/core/notes/javascripts/countly.views.js',
+                    'frontend/express/public/core/onboarding/javascripts/countly.models.js',
+                    'frontend/express/public/core/onboarding/javascripts/countly.views.js',
                 ],
                 dest: 'frontend/express/public/javascripts/min/countly.lib.concat.js'
             }

diff --git a/api/api.js b/api/api.js
@@ -80,7 +80,8 @@ plugins.connectToAllDatabases().then(function() {
         batch_read_processing: true,
         //batch_read_on_master: false,
         batch_read_ttl: 600,
-        batch_read_period: 60
+        batch_read_period: 60,
+        trim_trailing_ending_spaces: false
     });
 
     /**
@@ -255,6 +256,7 @@ plugins.connectToAllDatabases().then(function() {
     };
 
     if (cluster.isMaster) {
+        plugins.installMissingPlugins(common.db);
         common.runners = require('./parts/jobs/runner');
         common.cache = new CacheMaster(common.db);
         common.cache.start().then(() => {

diff --git a/api/jobs/ping.js b/api/jobs/ping.js
@@ -23,6 +23,7 @@ class PingJob extends job.Job {
         }
         plugins.loadConfigs(db, function() {
             const offlineMode = plugins.getConfig("api").offline_mode;
+            const { countly_tracking } = plugins.getConfig('frontend');
             if (!offlineMode) {
                 request(url, function(err, response, body) {
                     if (typeof body === "string") {
@@ -47,17 +48,28 @@ class PingJob extends job.Job {
                         }
                     }
                     log.d(err, body, countlyConfigOrig, countlyConfig);
-                    if (countlyConfig.web.track !== "none") {
+                    if (countly_tracking) {
                         db.collection("members").findOne({global_admin: true}, function(err2, member) {
                             if (!err2 && member) {
                                 var date = new Date();
+                                let domain = plugins.getConfig('api').domain;
+
+                                try {
+                                    // try to extract hostname from full domain url
+                                    const urlObj = new URL(domain);
+                                    domain = urlObj.hostname;
+                                }
+                                catch (_) {
+                                    // do nothing, domain from config will be used as is
+                                }
+
                                 request({
                                     uri: "https://stats.count.ly/i",
                                     method: "GET",
                                     timeout: 4E3,
                                     qs: {
-                                        device_id: member.email,
-                                        app_key: "386012020c7bf7fcb2f1edf215f1801d6146913f",
+                                        device_id: domain,
+                                        app_key: "e70ec21cbe19e799472dfaee0adb9223516d238f",
                                         timestamp: Math.floor(date.getTime() / 1000),
                                         hour: date.getHours(),
                                         dow: date.getDay(),

diff --git a/api/lib/countly.common.js b/api/lib/countly.common.js
@@ -2527,7 +2527,7 @@ countlyCommon.union = function(x, y) {
 * @returns {string} encoded string
 */
 countlyCommon.encode = function(str) {
-    return str.replace(/^\$/g, "&#36;").replace(/\./g, '&#46;');
+    return str.replace(/^\$/g, "&#36;").replace(/\./g, '&#46;').replace(/\u0000/g, "&#9647");
 };
 
 /**
@@ -2536,7 +2536,7 @@ countlyCommon.encode = function(str) {
 * @returns {string} decoded string
 */
 countlyCommon.decode = function(str) {
-    return str.replace(/^&#36;/g, "$").replace(/^&amp;#36;/g, '$').replace(/&#46;/g, '.').replace(/&amp;#46;/g, '.');
+    return str.replace(/^&#36;/g, "$").replace(/^&amp;#36;/g, '$').replace(/&#46;/g, '.').replace(/&amp;#46;/g, '.').replace(/&#9647/g, '\u0000');
 };
 
 /**

diff --git a/api/parts/data/exports.js b/api/parts/data/exports.js
@@ -424,8 +424,27 @@ exports.stream = function(params, stream, options) {
     }
 };
 
-
-
+/**
+ * Check if id is valid ObjectID
+ * @param {string} id - id to check
+ * @returns {boolean} true if valid
+ * */
+function isObjectId(id) {
+    var checkForHexRegExp = new RegExp("^[0-9a-fA-F]{24}$");
+    if (typeof id === "undefined" || id === null) {
+        return false;
+    }
+    if ((typeof id !== "undefined" && id !== null) && 'number' !== typeof id && (id.length !== 24)) {
+        return false;
+    }
+    else {
+        // Check specifically for hex correctness
+        if (typeof id === 'string' && id.length === 24) {
+            return checkForHexRegExp.test(id);
+        }
+        return true;
+    }
+}
 
 /**
 * Export data from database
@@ -471,7 +490,7 @@ exports.fromDatabase = function(options) {
         }
     }
     alternateName += "_exported_on_" + moment().format("DD-MMM-YYYY");
-    options.filename = options.filename || alternateName;
+    options.filename = options.filename || options.params.qstring.filename || alternateName;
 
     if (options.collection.startsWith("app_users")) {
         options.params.qstring.method = "user_details";
@@ -490,6 +509,9 @@ exports.fromDatabase = function(options) {
             query: options.query,
             params: options.params
         }, ()=>{
+            if (options.query._id && isObjectId(options.query._id)) {
+                options.query._id = common.db.ObjectID(options.query._id);
+            }
             var cursor = options.db.collection(options.collection).find(options.query, {"projection": options.projection});
             if (options.sort) {
                 cursor.sort(options.sort);

diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js
@@ -164,7 +164,7 @@ usage.setLocation = function(params) {
  */
 usage.setUserLocation = function(params, loc) {
     params.user.country = plugins.getConfig('api', params.app && params.app.plugins, true).country_data === false ? undefined : loc.country;
-    params.user.region = loc.region;
+    params.user.region = plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true ? loc.region : undefined;
     params.user.city = (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === false ||
         plugins.getConfig('api', params.app && params.app.plugins, true).country_data === false) ? undefined : loc.city;
 };
@@ -1011,7 +1011,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
                     userProps.cty = data.city;
                 }
 
-                if (!userProps.loc && typeof data.lat !== "undefined" && typeof data.lon !== "undefined") {
+                if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && typeof data.lat !== "undefined" && typeof data.lon !== "undefined") {
                     // only override lat/lon if no recent gps location exists in user document
                     if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
                         userProps.loc = {
@@ -1042,7 +1042,7 @@ plugins.register("/sdk/user_properties", async function(ob) {
                         userProps.cty = data.city;
                     }
 
-                    if (!userProps.loc && data.ll && typeof data.ll[0] !== "undefined" && typeof data.ll[1] !== "undefined") {
+                    if (plugins.getConfig('api', params.app && params.app.plugins, true).city_data === true && !userProps.loc && data.ll && typeof data.ll[0] !== "undefined" && typeof data.ll[1] !== "undefined") {
                         // only override lat/lon if no recent gps location exists in user document
                         if (!params.app_user.loc || (params.app_user.loc.gps && params.time.mstimestamp - params.app_user.loc.date > 7 * 24 * 3600)) {
                             userProps.loc = {

diff --git a/api/parts/jobs/job.js b/api/parts/jobs/job.js
@@ -513,7 +513,7 @@ class Job extends EventEmitter {
         }
 
         let existing = await Job.findMany(this.db(), query);
-        if (existing.length === 1) {
+        if (existing.length === 1 && (existing[0].schedule && existing[0].schedule === this._json.schedule)) {
             log.i('No need for replace of %s (%j): existing job %j', this.name, this.data, existing[0]);
             this._json = existing[0];
             return existing[0];

diff --git a/api/parts/mgmt/apps.js b/api/parts/mgmt/apps.js
@@ -58,8 +58,8 @@ appsApi.getCurrentUserApps = function(params) {
     var adminOfAppIds = getAdminApps(params.member),
         userOfAppIds = getUserApps(params.member);
 
-    common.db.collection('apps').find({ _id: { '$in': adminOfAppIds } }).toArray(function(err, admin_of) {
-        common.db.collection('apps').find({ _id: { '$in': userOfAppIds } }).toArray(function(err2, user_of) {
+    common.db.collection('apps').find({ _id: { '$in': adminOfAppIds.map(id => common.db.ObjectID(id)) } }).toArray(function(err, admin_of) {
+        common.db.collection('apps').find({ _id: { '$in': userOfAppIds.map(id => common.db.ObjectID(id)) } }).toArray(function(err2, user_of) {
             common.returnOutput(params, {
                 admin_of: packApps(admin_of),
                 user_of: packApps(user_of)
@@ -1021,7 +1021,8 @@ function packApps(apps) {
             'country': apps[i].country,
             'key': apps[i].key,
             'name': apps[i].name,
-            'timezone': apps[i].timezone
+            'timezone': apps[i].timezone,
+            'salt': apps[i].salt || apps[i].checksum_salt || "",
         };
     }
 

diff --git a/api/parts/mgmt/cms.js b/api/parts/mgmt/cms.js
@@ -28,7 +28,11 @@ function fetchFromCMS(params, callback) {
     * @param {number} pageNumber - page number
     **/
     function fetchPage(pageNumber) {
-        const pageUrl = `${url}?pagination[page]=${pageNumber}&pagination[pageSize]=${pageSize}`;
+        var pageUrl = `${url}?pagination[page]=${pageNumber}&pagination[pageSize]=${pageSize}`;
+
+        if (params.qstring.populate) {
+            pageUrl += `&populate=*`;
+        }
 
         fetch(pageUrl, {
             method: 'GET',

diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js
@@ -20,7 +20,7 @@ var tracker = {},
     asyncjs = require('async'),
     trial = "79199134e635edb05fc137e8cd202bb8640fb0eb",
     server = "e0693b48a5513cb60c112c21aede3cab809d52d0",
-    app = "386012020c7bf7fcb2f1edf215f1801d6146913f",
+    app = "e70ec21cbe19e799472dfaee0adb9223516d238f",
     url = "https://stats.count.ly",
     plugins = require('../../../plugins/pluginManager.js'),
     offlineMode = plugins.getConfig("api").offline_mode,

diff --git a/api/parts/mgmt/users.js b/api/parts/mgmt/users.js
@@ -468,7 +468,11 @@ usersApi.updateUser = async function(params) {
             'permission': {
                 'required': false,
                 'type': 'Object'
-            }
+            },
+            'subscribe_newsletter': {
+                'required': false,
+                'type': 'Boolean'
+            },
         },
         updatedMember = {},
         passwordNoHash = "";