Skip to content

Commit

Permalink
Merge pull request #102 from Countly/MemoryOnlyStorage
Browse files Browse the repository at this point in the history 
Memory Only Storage Option
  • Loading branch information
@turtledreams
turtledreams authored Oct 3, 2024
2 parents 97a4ccc + ae89395 commit 9f3f490
Show file tree
Hide file tree
Showing 8 changed files with 386 additions and 82 deletions.
6 changes: 5 additions & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
## xx.xx.xx
* Default max segmentation value count changed from 30 to 100
- Default max segmentation value count changed from 30 to 100
- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
- Added a new init time config option (conf.storage_type) which can make user set among these storage options:
- File Storage
- Memory Only Storage

## 22.06.0
- Fixed a bug where remote config requests were rejected
Expand Down
7 changes: 6 additions & 1 deletion lib/countly-bulk.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ var cc = require("./countly-common");
var BulkUser = require("./countly-bulk-user");
var CountlyStorage = require("./countly-storage");

const StorageTypes = cc.storageTypeEnums;

/**
* @lends module:lib/countly-bulk
* Initialize CountlyBulk server object
Expand All @@ -47,6 +49,7 @@ var CountlyStorage = require("./countly-storage");
* @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
* @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
* @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
* @example
* var server = new CountlyBulk({
* app_key: "{YOUR-API-KEY}",
Expand All @@ -72,6 +75,7 @@ function CountlyBulk(conf) {
var maxBreadcrumbCount = 100;
var maxStackTraceLinesPerThread = 30;
var maxStackTraceLineLength = 200;
var storageType = StorageTypes.FILE;

cc.debugBulk = conf.debug || false;
if (!conf.app_key) {
Expand Down Expand Up @@ -101,8 +105,9 @@ function CountlyBulk(conf) {
conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
conf.storage_type = conf.storage_type || storageType;

CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue);
CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue);

this.conf = conf;
/**
Expand Down
6 changes: 6 additions & 0 deletions lib/countly-common.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ var cc = {
ACTION: '[CLY]_action',
},

storageTypeEnums: {
FILE: "file",
MEMORY: "memory",
CUSTOM: "custom",
},

/**
* Get current timestamp
* @returns {number} unix timestamp in seconds
Expand Down
204 changes: 157 additions & 47 deletions lib/countly-storage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,133 @@ var defaultPath = "../data/"; // Default path
var defaultBulkPath = "../bulk_data/"; // Default path
var asyncWriteLock = false;
var asyncWriteQueue = [];
let storageMethod = {};
const StorageTypes = cc.storageTypeEnums;

// Memory-only storage methods
const memoryStorage = {
/**
* Save value in memory
* @param {String} key - key for value to store
* @param {varies} value - value to store
* @param {Function} callback - callback to call when done storing
*/
storeSet: function(key, value, callback) {
if (key) {
cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
__data[key] = value;
if (typeof callback === "function") {
callback(null);
}
}
else {
cc.log(cc.logLevelEnums.WARNING, `storeSet, Provioded key: [${key}] is null!`);
}
},
/**
* Get value from memory
* @param {String} key - key of value to get
* @param {varies} def - default value to use if not set
* @returns {varies} value for the key
*/
storeGet: function(key, def) {
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
return typeof __data[key] !== "undefined" ? __data[key] : def;
},
/**
* Remove value from memory
* @param {String} key - key of value to remove
*/
storeRemove: function(key) {
delete __data[key];
},
};

// File storage methods
const fileStorage = {
/**
* Save value in storage
* @param {String} key - key for value to store
* @param {varies} value - value to store
* @param {Function} callback - callback to call when done storing
*/
storeSet: function(key, value, callback) {
__data[key] = value;
if (!asyncWriteLock) {
asyncWriteLock = true;
writeFile(key, value, callback);
}
else {
asyncWriteQueue.push([key, value, callback]);
}
},
/**
* Get value from storage
* @param {String} key - key of value to get
* @param {varies} def - default value to use if not set
* @returns {varies} value for the key
*/
storeGet: function(key, def) {
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
if (typeof __data[key] === "undefined") {
var ob = readFile(key);
var obLen;
// check if the 'read object' is empty or not
try {
obLen = Object.keys(ob).length;
}
catch (error) {
// if we can not even asses length set it to 0 so we can return the default value
obLen = 0;
}

// if empty or falsy set default value
if (!ob || obLen === 0) {
__data[key] = def;
}
// else set the value read file has
else {
__data[key] = ob[key];
}
}
return __data[key];
},
storeRemove: function(key) {
delete __data[key];
var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
fs.access(filePath, fs.constants.F_OK, (accessErr) => {
if (accessErr) {
cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`);
return;
}
fs.unlink(filePath, (err) => {
if (err) {
cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`);
}
else {
cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`);
}
});
});
},
};

/**
* Sets the storage method, by default sets file storage and storage path.
* @param {String} userPath - User provided storage path
* @param {StorageTypes} storageType - Whether to use memory only storage or not
* @param {Boolean} isBulk - Whether the storage is for bulk data
* @param {Boolean} persistQueue - Whether to persist the queue until processed
*/
var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) {
if (storageType === StorageTypes.MEMORY) {
storageMethod = memoryStorage;
}
else {
storageMethod = fileStorage;
setStoragePath(userPath, isBulk, persistQueue);
}
};

/**
* Sets the storage path, defaulting to a specified path if none is provided.
Expand Down Expand Up @@ -63,15 +190,21 @@ var resetStorage = function() {
*/
var readFile = function(key) {
var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);

// try reading data file
var data;
try {
data = fs.readFileSync(dir);
data = fs.readFileSync(dir, 'utf8'); // read file as string
}
catch (ex) {
// there was no file, probably new init
data = null;
cc.log(cc.logLevelEnums.WARN, `readFile, File not found for key: [${key}]. Returning null.`);
return null; // early exit if file doesn't exist
}

// early exit if file is empty or whitespace
if (!data.trim()) {
cc.log(cc.logLevelEnums.WARN, `readFile, File is empty or contains only whitespace for key: [${key}]. Returning null.`);
return null;
}

try {
Expand All @@ -84,7 +217,7 @@ var readFile = function(key) {
// backup corrupted file data
fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { });
// start with new clean object
data = null;
return null; // return null in case of corrupted data
}
return data;
};
Expand Down Expand Up @@ -139,62 +272,39 @@ var writeFile = function(key, value, callback) {
});
};

/**
* Save value in storage
* @param {String} key - key for value to store
* @param {varies} value - value to store
* @param {Function} callback - callback to call when done storing
*/
var storeSet = function(key, value, callback) {
__data[key] = value;
if (!asyncWriteLock) {
asyncWriteLock = true;
writeFile(key, value, callback);
}
else {
asyncWriteQueue.push([key, value, callback]);
}
storageMethod.storeSet(key, value, callback);
};

/**
* Get value from storage
* @param {String} key - key of value to get
* @param {varies} def - default value to use if not set
* @returns {varies} value for the key
*/
var storeGet = function(key, def) {
cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
if (typeof __data[key] === "undefined") {
var ob = readFile(key);
var obLen;
// check if the 'read object' is empty or not
try {
obLen = Object.keys(ob).length;
}
catch (error) {
// if we can not even asses length set it to 0 so we can return the default value
obLen = 0;
}
return storageMethod.storeGet(key, def);
};

// if empty or falsy set default value
if (!ob || obLen === 0) {
__data[key] = def;
}
// else set the value read file has
else {
__data[key] = ob[key];
}
var storeRemove = function(key) {
storageMethod.storeRemove(key);
};

/**
* Disclaimer: This method is mainly for testing purposes.
* @returns {StorageTypes} Returns the active storage type for the SDK
*/
var getStorageType = function() {
if (storageMethod === memoryStorage) {
return StorageTypes.MEMORY;
}
return __data[key];
return StorageTypes.FILE;
};

module.exports = {
writeFile,
storeGet,
initStorage,
storeSet,
storeGet,
storeRemove,
writeFile,
forceStore,
getStoragePath,
setStoragePath,
resetStorage,
readFile,
getStorageType,
};
9 changes: 5 additions & 4 deletions lib/countly.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk");
var CountlyStorage = require("./countly-storage");

var Countly = {};
const StorageTypes = cc.storageTypeEnums;

Countly.Bulk = Bulk;
(function() {
Expand Down Expand Up @@ -71,7 +72,7 @@ Countly.Bulk = Bulk;
var maxStackTraceLinesPerThread = 30;
var maxStackTraceLineLength = 200;
var deviceIdType = null;

var storageType = StorageTypes.FILE;
/**
* Array with list of available features that you can require consent for
*/
Expand Down Expand Up @@ -122,6 +123,7 @@ Countly.Bulk = Bulk;
* @param {string} conf.metrics._density - screen density of the device
* @param {string} conf.metrics._locale - locale or language of the device in ISO format
* @param {string} conf.metrics._store - source from where the user/device/installation came from
* @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
* @example
* Countly.init({
* app_key: "{YOUR-APP-KEY}",
Expand Down Expand Up @@ -166,12 +168,11 @@ Countly.Bulk = Bulk;
Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;

conf.storage_type = conf.storage_type || storageType;
// Common module debug value is set to init time debug value
cc.debug = conf.debug;

// Set the storage path
CountlyStorage.setStoragePath(conf.storage_path);
CountlyStorage.initStorage(conf.storage_path, conf.storage_type);

// clear stored device ID if flag is set
if (conf.clear_stored_device_id) {
Expand Down
Loading

0 comments on commit 9f3f490

Please sign in to comment.