Skip to content

Commit

Permalink
Merge pull request #104 from Countly/CustomStorageOption
Browse files Browse the repository at this point in the history 
Custom Storage Option
  • Loading branch information
@turtledreams
turtledreams authored Oct 21, 2024
2 parents d245d64 + 7c5c523 commit c966016
Show file tree
Hide file tree
Showing 17 changed files with 1,064 additions and 762 deletions.
7 changes: 4 additions & 3 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
## xx.xx.xx
## 24.10.0
- Default max segmentation value count changed from 30 to 100
- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
- Added a new init time config option (conf.storage_type) which can make user set among these storage options:
- Mitigated an issue where SDK could create an unintended dump file
- Added a new init time config option (conf.storage_type) which can make user set the SDK storage option:
- File Storage
- Memory Only Storage
- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods

## 22.06.0
- Fixed a bug where remote config requests were rejected
Expand Down
36 changes: 30 additions & 6 deletions lib/countly-bulk.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ var cc = require("./countly-common");
var BulkUser = require("./countly-bulk-user");
var CountlyStorage = require("./countly-storage");

const StorageTypes = cc.storageTypeEnums;
CountlyBulk.StorageTypes = cc.storageTypeEnums;

/**
* @lends module:lib/countly-bulk
Expand All @@ -39,7 +39,6 @@ const StorageTypes = cc.storageTypeEnums;
* @param {number} [conf.fail_timeout=60] - set time in seconds to wait after failed connection to server in seconds
* @param {number} [conf.session_update=60] - how often in seconds should session be extended
* @param {number} [conf.max_events=100] - maximum amount of events to send in one batch
* @param {boolean} [conf.persist_queue=false] - persistently store queue until processed, default is false if you want to keep queue in memory and process all in one process run
* @param {boolean} [conf.force_post=false] - force using post method for all requests
* @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc
* @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request
Expand All @@ -50,6 +49,8 @@ const StorageTypes = cc.storageTypeEnums;
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
* @param {Object} conf.custom_storage_method - user given storage methods
* @param {boolean} [conf.persist_queue=false] - *DEPRECATED* persistent mode instead of using in-memory queue. Use storage_type and storage_path instead
* @example
* var server = new CountlyBulk({
* app_key: "{YOUR-API-KEY}",
Expand All @@ -75,8 +76,6 @@ function CountlyBulk(conf) {
var maxBreadcrumbCount = 100;
var maxStackTraceLinesPerThread = 30;
var maxStackTraceLineLength = 200;
var storageType = StorageTypes.FILE;

cc.debugBulk = conf.debug || false;
if (!conf.app_key) {
cc.log(cc.logLevelEnums.ERROR, "CountlyBulk, 'app_key' is missing.");
Expand Down Expand Up @@ -105,9 +104,13 @@ function CountlyBulk(conf) {
conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
conf.storage_type = conf.storage_type || storageType;

CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue);
// bulk mode is memory only by default
if (typeof conf.storage_type === "undefined" && conf.persist_queue === false) {
conf.storage_type = CountlyBulk.StorageTypes.MEMORY;
}

CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.custom_storage_method);

this.conf = conf;
/**
Expand Down Expand Up @@ -599,6 +602,27 @@ function CountlyBulk(conf) {
var requestQueue = CountlyStorage.storeGet("cly_req_queue", []);
var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {});
var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []);
/**
* getBulkEventQueue is a testing purposed method which returns the event queue object
* @returns {Object} eventQueue
*/
this._getBulkEventQueue = function() {
return eventQueue;
};
/**
* getBulkRequestQueue is a testing purposed method which returns the request queue object
* @returns {Object} requestQueue
*/
this._getBulkRequestQueue = function() {
return requestQueue;
};
/**
* getBulkQueue is a testing purposed method which returns the bulk queue object
* @returns {Object} bulkQueue
*/
this._getBulkQueue = function() {
return bulkQueue;
};
}

module.exports = CountlyBulk;
1 change: 0 additions & 1 deletion lib/countly-common.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ var cc = {
storageTypeEnums: {
FILE: "file",
MEMORY: "memory",
CUSTOM: "custom",
},

/**
Expand Down
110 changes: 74 additions & 36 deletions lib/countly-storage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,50 @@ const fs = require('fs');
const path = require('path');
var cc = require("./countly-common");

// Constants
const defaultPath = "../data/"; // Default storage path
const defaultBulkPath = "../bulk_data/"; // Default bulk storage path
const StorageTypes = cc.storageTypeEnums;
const defaultStorageType = StorageTypes.FILE;
const customTypeName = "custom";

var storagePath;
var __data = {};
var defaultPath = "../data/"; // Default path
var defaultBulkPath = "../bulk_data/"; // Default path
let storageMethod = {};
var __cache = {};
var asyncWriteLock = false;
var asyncWriteQueue = [];
let storageMethod = {};
const StorageTypes = cc.storageTypeEnums;

/**
* Sets the storage method, by default sets file storage and storage path.
* @param {String} userPath - User provided storage path
* @param {StorageTypes} storageType - Whether to use memory only storage or not
* @param {Boolean} isBulk - Whether the storage is for bulk data
* @param {varies} customStorageMethod - Storage methods provided by the user
*/
var initStorage = function(userPath, storageType, isBulk = false, customStorageMethod = null) {
cc.log(cc.logLevelEnums.INFO, `Initializing storage with userPath: [${userPath}], storageType: [${storageType}], isBulk: [${isBulk}], customStorageMethod type: [${typeof customStorageMethod}].`);

// set storage type
storageType = storageType || defaultStorageType;
storageMethod = fileStorage; // file storage is default

if (storageType === StorageTypes.MEMORY) {
storageMethod = memoryStorage;
cc.log(cc.logLevelEnums.DEBUG, `Using memory storage!`);
}

// at this point we either use memory or file storage. If custom storage is provided, check if it is valid and use it instead
if (isCustomStorageValid(customStorageMethod)) {
storageMethod = customStorageMethod;
storageType = customTypeName;
cc.log(cc.logLevelEnums.DEBUG, `Using custom storage!`);
}

// set storage path if not memory storage
if (storageType !== StorageTypes.MEMORY) {
setStoragePath(userPath, isBulk);
}
};

// Memory-only storage methods
const memoryStorage = {
Expand All @@ -22,7 +58,7 @@ const memoryStorage = {
storeSet: function(key, value, callback) {
if (key) {
cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
__data[key] = value;
__cache[key] = value;
if (typeof callback === "function") {
callback(null);
}
Expand All @@ -39,14 +75,14 @@ const memoryStorage = {
*/
storeGet: function(key, def) {
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
return typeof __data[key] !== "undefined" ? __data[key] : def;
return typeof __cache[key] !== "undefined" ? __cache[key] : def;
},
/**
* Remove value from memory
* @param {String} key - key of value to remove
*/
storeRemove: function(key) {
delete __data[key];
delete __cache[key];
},
};

Expand All @@ -59,7 +95,7 @@ const fileStorage = {
* @param {Function} callback - callback to call when done storing
*/
storeSet: function(key, value, callback) {
__data[key] = value;
__cache[key] = value;
if (!asyncWriteLock) {
asyncWriteLock = true;
writeFile(key, value, callback);
Expand All @@ -76,7 +112,7 @@ const fileStorage = {
*/
storeGet: function(key, def) {
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
if (typeof __data[key] === "undefined") {
if (typeof __cache[key] === "undefined") {
var ob = readFile(key);
var obLen;
// check if the 'read object' is empty or not
Expand All @@ -90,17 +126,17 @@ const fileStorage = {

// if empty or falsy set default value
if (!ob || obLen === 0) {
__data[key] = def;
__cache[key] = def;
}
// else set the value read file has
else {
__data[key] = ob[key];
__cache[key] = ob[key];
}
}
return __data[key];
return __cache[key];
},
storeRemove: function(key) {
delete __data[key];
delete __cache[key];
var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
fs.access(filePath, fs.constants.F_OK, (accessErr) => {
if (accessErr) {
Expand All @@ -119,35 +155,31 @@ const fileStorage = {
},
};

/**
* Sets the storage method, by default sets file storage and storage path.
* @param {String} userPath - User provided storage path
* @param {StorageTypes} storageType - Whether to use memory only storage or not
* @param {Boolean} isBulk - Whether the storage is for bulk data
* @param {Boolean} persistQueue - Whether to persist the queue until processed
*/
var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) {
if (storageType === StorageTypes.MEMORY) {
storageMethod = memoryStorage;
var isCustomStorageValid = function(storage) {
if (!storage) {
return false;
}
if (typeof storage.storeSet !== 'function') {
return false;
}
if (typeof storage.storeGet !== 'function') {
return false;
}
else {
storageMethod = fileStorage;
setStoragePath(userPath, isBulk, persistQueue);
if (typeof storage.storeRemove !== 'function') {
return false;
}
return true;
};

/**
* Sets the storage path, defaulting to a specified path if none is provided.
* @param {String} userPath - User provided storage path
* @param {Boolean} isBulk - Whether the storage is for bulk data
* @param {Boolean} persistQueue - Whether to persist the queue until processed
*/
var setStoragePath = function(userPath, isBulk = false, persistQueue = false) {
var setStoragePath = function(userPath, isBulk = false) {
storagePath = userPath || (isBulk ? defaultBulkPath : defaultPath);

if (!isBulk || persistQueue) {
createDirectory(path.resolve(__dirname, storagePath));
}
createDirectory(path.resolve(__dirname, storagePath));
};

/**
Expand Down Expand Up @@ -178,9 +210,10 @@ var createDirectory = function(dir) {
*/
var resetStorage = function() {
storagePath = undefined;
__data = {};
__cache = {};
asyncWriteLock = false;
asyncWriteQueue = [];
storageMethod = {};
};

/**
Expand Down Expand Up @@ -226,10 +259,10 @@ var readFile = function(key) {
* Force store data synchronously on unrecoverable errors to preserve it for next launch
*/
var forceStore = function() {
for (var i in __data) {
for (var i in __cache) {
var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`);
var ob = {};
ob[i] = __data[i];
ob[i] = __cache[i];
try {
fs.writeFileSync(dir, JSON.stringify(ob));
}
Expand Down Expand Up @@ -292,7 +325,12 @@ var getStorageType = function() {
if (storageMethod === memoryStorage) {
return StorageTypes.MEMORY;
}
return StorageTypes.FILE;

if (storageMethod === fileStorage) {
return StorageTypes.FILE;
}

return null;
};

module.exports = {
Expand Down
18 changes: 12 additions & 6 deletions lib/countly.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ var Bulk = require("./countly-bulk");
var CountlyStorage = require("./countly-storage");

var Countly = {};
const StorageTypes = cc.storageTypeEnums;

Countly.StorageTypes = cc.storageTypeEnums;
Countly.Bulk = Bulk;
(function() {
var SDK_VERSION = "24.10.0";
Expand Down Expand Up @@ -72,7 +71,7 @@ Countly.Bulk = Bulk;
var maxStackTraceLinesPerThread = 30;
var maxStackTraceLineLength = 200;
var deviceIdType = null;
var storageType = StorageTypes.FILE;
var heartBeatTimer = null;
/**
* Array with list of available features that you can require consent for
*/
Expand Down Expand Up @@ -124,6 +123,7 @@ Countly.Bulk = Bulk;
* @param {string} conf.metrics._locale - locale or language of the device in ISO format
* @param {string} conf.metrics._store - source from where the user/device/installation came from
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
* @param {Object} conf.custom_storage_method - user given storage methods
* @example
* Countly.init({
* app_key: "{YOUR-APP-KEY}",
Expand Down Expand Up @@ -168,11 +168,12 @@ Countly.Bulk = Bulk;
Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;
conf.storage_type = conf.storage_type || storageType;
conf.storage_path = conf.storage_path || Countly.storage_path;
conf.storage_type = conf.storage_type || Countly.storage_type;
// Common module debug value is set to init time debug value
cc.debug = conf.debug;

CountlyStorage.initStorage(conf.storage_path, conf.storage_type);
CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, conf.custom_storage_method);

// clear stored device ID if flag is set
if (conf.clear_stored_device_id) {
Expand Down Expand Up @@ -320,6 +321,10 @@ Countly.Bulk = Bulk;
maxStackTraceLinesPerThread = 30;
maxStackTraceLineLength = 200;
deviceIdType = null;
if (heartBeatTimer) {
clearInterval(heartBeatTimer);
heartBeatTimer = null;
}

// cc DEBUG
cc.debug = false;
Expand All @@ -334,6 +339,7 @@ Countly.Bulk = Bulk;

// device_id
Countly.device_id = undefined;
Countly.device_id_type = undefined;
Countly.remote_config = undefined;
Countly.require_consent = false;
Countly.debug = undefined;
Expand Down Expand Up @@ -1478,7 +1484,7 @@ Countly.Bulk = Bulk;
}, "heartBeat", false);
}

setTimeout(heartBeat, beatInterval);
heartBeatTimer = setTimeout(heartBeat, beatInterval);
}

/**
Expand Down
Loading

0 comments on commit c966016

Please sign in to comment.