Some changes

CHANGELOG.md

@@ -1,11 +1,10 @@
-## xx.xx.xx
+## 24.10.0
 - Default max segmentation value count changed from 30 to 100
-- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
-- Added a new init time config option (conf.storage_type) which can make user set among these storage options:
+- Mitigated an issue where SDK could create an unintended dump file
+- Added a new init time config option (conf.storage_type) which can make user set the SDK storage option:
   - File Storage
   - Memory Only Storage
-  - Custom Storage Methods
-- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods.
+- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods
 
 ## 22.06.0
 - Fixed a bug where remote config requests were rejected

lib/countly-bulk.js

@@ -39,7 +39,7 @@ const StorageTypes = cc.storageTypeEnums;
  * @param {number} [conf.fail_timeout=60] - set time in seconds to wait after failed connection to server in seconds
  * @param {number} [conf.session_update=60] - how often in seconds should session be extended
  * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch
- * @param {boolean} [conf.persist_queue=false] - persistently store queue until processed, default is false if you want to keep queue in memory and process all in one process run
+ * @deprecated {boolean} [conf.persist_queue=false] - persistent mode instead of using in-memory queue. Use storage_type and storage_path instead
  * @param {boolean} [conf.force_post=false] - force using post method for all requests
  * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc
  * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request
@@ -50,7 +50,7 @@ const StorageTypes = cc.storageTypeEnums;
  * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
  * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
  * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
- * @param {varies} conf.custom_storage_method - user given storage methods
+ * @param {Object} conf.custom_storage_method - user given storage methods
  * @example
  * var server = new CountlyBulk({
  *   app_key: "{YOUR-API-KEY}",
@@ -76,7 +76,6 @@ function CountlyBulk(conf) {
     var maxBreadcrumbCount = 100;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
-    var storageType = StorageTypes.FILE;
 
     cc.debugBulk = conf.debug || false;
     if (!conf.app_key) {
@@ -106,9 +105,13 @@ function CountlyBulk(conf) {
     conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
     conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
     conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
-    conf.storage_type = conf.storage_type || storageType;
 
-    CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue, conf.custom_storage_method);
+    // bulk mode is memory only by default
+    if (typeof conf.storage_type === "undefined" && conf.persist_queue === false) {
+        conf.storage_type = StorageTypes.MEMORY;
+    }
+
+    CountlyStorage.initStorage(conf.storage_path, conf.storage_type, conf.custom_storage_method);
 
     this.conf = conf;
     /**

lib/countly-storage.js

@@ -2,14 +2,50 @@ const fs = require('fs');
 const path = require('path');
 var cc = require("./countly-common");
 
+// Constants
+const defaultPath = "../data/"; // Default storage path
+const defaultBulkPath = "../bulk_data/"; // Default bulk storage path
+const StorageTypes = cc.storageTypeEnums;
+const defaultStorageType = StorageTypes.FILE;
+const customTypeName = "custom";
+
 var storagePath;
-var __data = {};
-var defaultPath = "../data/"; // Default path
-var defaultBulkPath = "../bulk_data/"; // Default path
+let storageMethod = {};
+var __cache = {};
 var asyncWriteLock = false;
 var asyncWriteQueue = [];
-let storageMethod = {};
-const StorageTypes = cc.storageTypeEnums;
+
+/**
+ * Sets the storage method, by default sets file storage and storage path.
+ * @param {String} userPath - User provided storage path
+ * @param {StorageTypes} storageType - Whether to use memory only storage or not
+ * @param {Boolean} isBulk - Whether the storage is for bulk data
+ * @param {varies} customStorageMethod - Storage methods provided by the user
+ */
+var initStorage = function(userPath, storageType, isBulk = false, customStorageMethod = null) {
+    cc.log(cc.logLevelEnums.INFO, `Initializing storage with userPath: [${userPath}], storageType: [${storageType}], isBulk: [${isBulk}], customStorageMethod type: [${typeof customStorageMethod}].`);
+
+    // set storage type
+    storageType = storageType || defaultStorageType;
+    storageMethod = fileStorage; // file storage is default
+
+    if (storageType === StorageTypes.MEMORY) {
+        storageMethod = memoryStorage;
+        cc.log(cc.logLevelEnums.DEBUG, `Using memory storage!`);
+    }
+
+    // at this point we either use memory or file storage. If custom storage is provided, check if it is valid and use it instead
+    if (isCustomStorageValid(customStorageMethod)) {
+        storageMethod = customStorageMethod;
+        storageType = customTypeName;
+        cc.log(cc.logLevelEnums.DEBUG, `Using custom storage!`);
+    }
+
+    // set storage path if not memory storage
+    if (storageType !== StorageTypes.MEMORY) {
+        setStoragePath(userPath, isBulk);
+    }
+};
 
 // Memory-only storage methods
 const memoryStorage = {
@@ -22,7 +58,7 @@ const memoryStorage = {
     storeSet: function(key, value, callback) {
         if (key) {
             cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
-            __data[key] = value;
+            __cache[key] = value;
             if (typeof callback === "function") {
                 callback(null);
             }
@@ -39,14 +75,14 @@ const memoryStorage = {
      */
     storeGet: function(key, def) {
         cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
-        return typeof __data[key] !== "undefined" ? __data[key] : def;
+        return typeof __cache[key] !== "undefined" ? __cache[key] : def;
     },
     /**
      *  Remove value from memory
      *  @param {String} key - key of value to remove
      */
     storeRemove: function(key) {
-        delete __data[key];
+        delete __cache[key];
     },
 };
 
@@ -59,7 +95,7 @@ const fileStorage = {
      *  @param {Function} callback - callback to call when done storing
      */
     storeSet: function(key, value, callback) {
-        __data[key] = value;
+        __cache[key] = value;
         if (!asyncWriteLock) {
             asyncWriteLock = true;
             writeFile(key, value, callback);
@@ -76,7 +112,7 @@ const fileStorage = {
      */
     storeGet: function(key, def) {
         cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
-        if (typeof __data[key] === "undefined") {
+        if (typeof __cache[key] === "undefined") {
             var ob = readFile(key);
             var obLen;
             // check if the 'read object' is empty or not
@@ -90,17 +126,17 @@ const fileStorage = {
 
             // if empty or falsy set default value
             if (!ob || obLen === 0) {
-                __data[key] = def;
+                __cache[key] = def;
             }
             // else set the value read file has
             else {
-                __data[key] = ob[key];
+                __cache[key] = ob[key];
             }
         }
-        return __data[key];
+        return __cache[key];
     },
     storeRemove: function(key) {
-        delete __data[key];
+        delete __cache[key];
         var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
         fs.access(filePath, fs.constants.F_OK, (accessErr) => {
             if (accessErr) {
@@ -119,38 +155,7 @@ const fileStorage = {
     },
 };
 
-/**
- * Sets the storage method, by default sets file storage and storage path.
- * @param {String} userPath - User provided storage path
- * @param {StorageTypes} storageType - Whether to use memory only storage or not
- * @param {Boolean} isBulk - Whether the storage is for bulk data
- * @param {Boolean} persistQueue - Whether to persist the queue until processed
- * @param {varies} customStorageMethod - Storage methods provided by the user
- */
-var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false, customStorageMethod = null) {
-    if (storageType === StorageTypes.MEMORY) {
-        storageMethod = memoryStorage;
-    }
-    else if (customStorageMethod) {
-        if (hasValidMethods(customStorageMethod)) {
-            storageMethod = customStorageMethod;
-            if (userPath) {
-                setStoragePath(userPath, isBulk, persistQueue);
-            }
-        }
-        else {
-            cc.log(cc.logLevelEnums.WARNING, `Provided Custom Storage Methods are not valid. Switching to default file storage!`);
-            storageMethod = fileStorage;
-            setStoragePath(userPath, isBulk, persistQueue);
-        }
-    }
-    else {
-        storageMethod = fileStorage;
-        setStoragePath(userPath, isBulk, persistQueue);
-    }
-};
-
-var hasValidMethods = function(storage) {
+var isCustomStorageValid = function(storage) {
     if (!storage) {
         return false;
     }
@@ -170,14 +175,11 @@ var hasValidMethods = function(storage) {
  * Sets the storage path, defaulting to a specified path if none is provided.
  * @param {String} userPath - User provided storage path
  * @param {Boolean} isBulk - Whether the storage is for bulk data
- * @param {Boolean} persistQueue - Whether to persist the queue until processed
  */
-var setStoragePath = function(userPath, isBulk = false, persistQueue = false) {
+var setStoragePath = function(userPath, isBulk = false) {
     storagePath = userPath || (isBulk ? defaultBulkPath : defaultPath);
 
-    if (!isBulk || persistQueue) {
-        createDirectory(path.resolve(__dirname, storagePath));
-    }
+    createDirectory(path.resolve(__dirname, storagePath));
 };
 
 /**
@@ -208,9 +210,10 @@ var createDirectory = function(dir) {
  */
 var resetStorage = function() {
     storagePath = undefined;
-    __data = {};
+    __cache = {};
     asyncWriteLock = false;
     asyncWriteQueue = [];
+    storageMethod = {};
 };
 
 /**
@@ -256,10 +259,10 @@ var readFile = function(key) {
  *  Force store data synchronously on unrecoverable errors to preserve it for next launch
  */
 var forceStore = function() {
-    for (var i in __data) {
+    for (var i in __cache) {
         var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`);
         var ob = {};
-        ob[i] = __data[i];
+        ob[i] = __cache[i];
         try {
             fs.writeFileSync(dir, JSON.stringify(ob));
         }

lib/countly.js

@@ -29,7 +29,6 @@ var Bulk = require("./countly-bulk");
 var CountlyStorage = require("./countly-storage");
 
 var Countly = {};
-const StorageTypes = cc.storageTypeEnums;
 
 Countly.Bulk = Bulk;
 (function() {
@@ -72,7 +71,6 @@ Countly.Bulk = Bulk;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
     var deviceIdType = null;
-    var storageType = StorageTypes.FILE;
     /**
     * Array with list of available features that you can require consent for
     */
@@ -124,7 +122,7 @@ Countly.Bulk = Bulk;
  * @param {string} conf.metrics._locale - locale or language of the device in ISO format
  * @param {string} conf.metrics._store - source from where the user/device/installation came from
  * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
- * @param {varies} conf.custom_storage_method - user given storage methods
+ * @param {Object} conf.custom_storage_method - user given storage methods
  * @example
  * Countly.init({
  *   app_key: "{YOUR-APP-KEY}",
@@ -169,11 +167,12 @@ Countly.Bulk = Bulk;
             Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
             Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
             Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;
-            conf.storage_type = conf.storage_type || storageType;
+            conf.storage_path = conf.storage_path || Countly.storage_path;
+            conf.storage_type = conf.storage_type || Countly.storage_type;
             // Common module debug value is set to init time debug value
             cc.debug = conf.debug;
 
-            CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, false, conf.custom_storage_method);
+            CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, conf.custom_storage_method);
 
             // clear stored device ID if flag is set
             if (conf.clear_stored_device_id) {

test/helpers/helper_functions.js

@@ -3,20 +3,23 @@ var path = require("path");
 var assert = require("assert");
 var fs = require("fs");
 var Countly = require("../../lib/countly");
-var CountlyStorage = require("../../lib/countly-storage");
 
 // paths for convenience
 var dir = path.resolve(__dirname, "../../");
+var defaultStoragePath = (`${dir}/data`);
+var defaultBulkStoragePath = (`${dir}/bulk_data`);
 var idDir = (`${dir}/data/__cly_id.json`);
 var idTypeDir = (`${dir}/data/__cly_id_type.json`);
 var eventDir = (`${dir}/data/__cly_event.json`);
 var reqDir = (`${dir}/data/__cly_queue.json`);
 var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`);
 var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`);
+
 // timeout variables
 var sWait = 50;
 var mWait = 3000;
 var lWait = 10000;
+
 // parsing event queue
 function readEventQueue(givenPath = null, isBulk = false) {
     var destination = eventDir;
@@ -52,35 +55,24 @@ function doesFileStoragePathsExist(callback) {
         });
     });
 }
-function clearStorage(keepID = false, isBulk = false, customPath = null) {
+function clearStorage(customPath = null) {
     Countly.halt(true);
-    const eventDirectory = isBulk ? bulkEventDir : eventDir;
-    const reqDirectory = isBulk ? bulkQueueDir : reqDir;
-    function removeDir(directory) {
-        const filePath = path.resolve(__dirname, `${directory}`);
-        if (fs.existsSync(filePath)) {
-            fs.rmSync(filePath, { recursive: true, force: true });
-        }
+    if (fs.existsSync(defaultStoragePath)) {
+        fs.rmSync(defaultStoragePath, { recursive: true, force: true });
+    }
+    if (fs.existsSync(defaultBulkStoragePath)) {
+        fs.rmSync(defaultBulkStoragePath, { recursive: true, force: true });
     }
-    removeDir(eventDirectory);
-    removeDir(reqDirectory);
-    if (!keepID) {
-        removeDir(idDir);
-        removeDir(idTypeDir);
+    if (customPath !== null && typeof customPath === 'string' && fs.existsSync(customPath)) {
+        fs.rmSync(customPath, { recursive: true, force: true });
     }
 
-    if (customPath) {
-        if (!keepID) {
-            removeDir(path.join(customPath, '__cly_id.json'));
-            removeDir(path.join(customPath, '__cly_id_type.json'));
-        }
-        removeDir(path.join(customPath, '__cly_event.json'));
-        removeDir(path.join(customPath, '__cly_queue.json'));
+    // make sure the directories are removed
+    if (fs.existsSync(defaultStoragePath) || fs.existsSync(defaultBulkStoragePath) || (customPath !== null && fs.existsSync(customPath))) {
+        throw new Error("Failed to clear storage");
     }
-    return new Promise((resolve) => {
-        resolve("Storage cleared");
-    });
 }
+
 /**
  * bunch of tests specifically gathered for testing events
  * @param {Object} eventObject - Original event object to test