Skip to content

Commit

Permalink
reorganize export data
Browse files Browse the repository at this point in the history
  • Loading branch information
@cs0x7f
cs0x7f committed Aug 3, 2024
1 parent 99183b3 commit 561b9d9
Show file tree
Hide file tree
Showing 2 changed files with 254 additions and 23 deletions.
154 changes: 154 additions & 0 deletions dist/userdata2.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
<?php
define('CSTIMER_USERDATA_LOGFILE', 'logfile');

if (!isset($_POST['id']) || empty($_POST['id']) || strlen($_POST['id']) >= 250) {
echo '{"retcode":400,"reason":"invalid uid"}';
exit(0);
}
if (!preg_match("/^[a-zA-Z0-9]+$/", $_POST['id'])) {
echo '{"retcode":400,"reason":"invalid uid"}';
exit(0);
}
if (!isset($_POST['offset']) || empty($_POST['offset'])) {
$offset = '0';
} else if (!preg_match("/^[0-9]+$/", $_POST['offset']) || strlen($_POST['offset']) > 5) {
echo '{"retcode":400,"reason":"invalid offset"}';
exit(0);
} else {
$offset = $_POST['offset'];
}
header("Access-Control-Allow-Origin: *");
$uid = $_POST['id'];

$db = new mysqli('localhost', 'cstimer', '', 'cstimer');
if ($db->connect_errno) {
echo '{"retcode":500,"reason":"db connect error"}';
die('Could not connect: ' . $db->connect_error);
}

if (isset($_POST['data'])) {//SET
if (!preg_match("/^[a-zA-Z0-9+\-]+$/", $_POST['data'])) {
echo '{"retcode":400,"reason":"invalid data"}';
exit(0);
}
$data = $_POST['data'];
error_log("[" . date("Y-m-d H:i:sO") . "] SET $uid " . strlen($data) . "\n", 3, CSTIMER_USERDATA_LOGFILE);
$data_md5 = md5($data);
$data_len = strlen($data);
$sql = "INSERT INTO `export_data2` (`uid`, `value_hash`, `value_len`, `value`) VALUES ('$uid', '$data_md5', '$data_len', '$data') ON DUPLICATE KEY UPDATE `upload_time` = CURRENT_TIMESTAMP";
$ret = $db->query($sql);
if ($ret === true) {
echo '{"retcode":0}';
} else {
echo '{"retcode":500,"reason":"db insert error"}';
}
} else if (isset($_POST['cnt'])) {
$sql = "SELECT `value_len` AS size,
unix_timestamp(`upload_time`) AS modifiedTime
FROM (
SELECT `value_len`, `upload_time` FROM `export_data` WHERE `uid` = '$uid'
UNION ALL
SELECT 0, `upload_time` FROM `export_data2` WHERE `uid` = '$uid'
) t1 ORDER BY `upload_time` DESC";
$ret = $db->query($sql);
if ($ret === false) {
echo '{"retcode":500,"reason":"db select error"}';
exit(0);
}
$arr = array();
while($row = $ret->fetch_assoc()) {
$row['modifiedTime'] = date("c", $row['modifiedTime']);
$arr[] = $row;
}
$ret = json_encode($arr);
error_log("[" . date("Y-m-d H:i:sO") . "] CNT $uid " . $ret . "\n", 3, CSTIMER_USERDATA_LOGFILE);
echo '{"retcode":0,"data":"' . count($arr) . '","files":' . $ret . '}';
} else if (isset($_POST['datas'])) {//SETMUL
if (!preg_match("/^[a-zA-Z0-9,]+$/", $_POST['ids'])) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
$ids = explode(",", $_POST['ids'], 4096);
if (count($ids) >= 4096) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
if (!preg_match("/^[a-zA-Z0-9+\-,]+$/", $_POST['datas'])) {
echo '{"retcode":400,"reason":"invalid data"}';
exit(0);
}
$datas = explode(",", $_POST['datas'], 4096);
if (count($datas) != count($ids)) {
echo '{"retcode":400,"reason":"invalid datas"}';
exit(0);
}
foreach ($ids as $id1) {
if (strlen($id1) >= 250 || strlen($id1) < 1) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
}
error_log("[" . date("Y-m-d H:i:sO") . "] SETMUL $uid " . count($ids) . " datas\n", 3, CSTIMER_USERDATA_LOGFILE);
$sql = "INSERT INTO `export_data2` (`uid`, `value_hash`, `value_len`, `value`) VALUES ";
for ($i1 = 0; $i1 < count($ids); $i1++) {
$data = $datas[$i1];
$data_md5 = md5($data);
$data_len = strlen($data);
$sql = $sql . ($i1 == 0 ? "" : ",") . "('$ids[$i1]', '$data_md5', '$data_len', '$data')";
}
$sql = $sql . " ON DUPLICATE KEY UPDATE `upload_time` = CURRENT_TIMESTAMP";
$ret = $db->query($sql);
if ($ret === true) {
echo '{"retcode":0}';
} else {
echo '{"retcode":500,"reason":"db insert error"}';
}
} else if (isset($_POST['ids'])) {//GETMUL
if (!preg_match("/^[a-zA-Z0-9,]+$/", $_POST['ids'])) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
$ids = explode(",", $_POST['ids'], 4096);
if (count($ids) >= 4096) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
foreach ($ids as $id1) {
if (strlen($id1) >= 250 || strlen($id1) < 1) {
echo '{"retcode":400,"reason":"invalid ids"}';
exit(0);
}
}
$sql = "SELECT `uid`, `value` FROM `export_data2` WHERE `uid` in ('" . join("','", $ids) . "')";
$ret = $db->query($sql);
if ($ret === false) {
echo '{"retcode":500,"reason":"db select error"}';
exit(0);
}
$arr = array();
while($row = $ret->fetch_assoc()) {
$arr[$row['uid']] = $row['value'];
}
error_log("[" . date("Y-m-d H:i:sO") . "] GETMUL $uid " . count($ids) . "\n", 3, CSTIMER_USERDATA_LOGFILE);
echo '{"retcode":0,"datas":' . json_encode($arr) . '}';
} else {//GET
$sql = "SELECT `value`
FROM (
SELECT `value`, `upload_time` FROM `export_data` WHERE `uid` = '$uid'
UNION ALL
SELECT `value`, `upload_time` FROM `export_data2` WHERE `uid` = '$uid'
) t1 ORDER BY `upload_time` DESC LIMIT $offset,1;";
$ret = $db->query($sql);
if ($ret === false) {
echo '{"retcode":500,"reason":"db select error"}';
exit(0);
}
if ($ret->num_rows == 0) {
echo '{"retcode":404,"reason":"not found"}';
exit(0);
}
$ret = $ret->fetch_assoc()['value'];
error_log("[" . date("Y-m-d H:i:sO") . "] GET $uid " . strlen($ret) . " $offset\n", 3, CSTIMER_USERDATA_LOGFILE);
echo '{"retcode":0,"data":"' . $ret . '"}';
}
?>
123 changes: 100 additions & 23 deletions src/js/export.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,18 +142,46 @@ var exportFunc = execMain(function() {
}

function uploadData(id) {
return new Promise(function(resolve, reject) {
var compExpString = LZString.compressToEncodedURIComponent(expString);
$.post('https://cstimer.net/userdata.php', {
'id': id,
'data': compExpString
}, function(val) {
if (val['retcode'] == 0) {
resolve(val);
} else {
reject(val);
var slices = {};
var baseObj = {};
baseObj['properties'] = mathlib.str2obj(localStorage['properties']);
return storage.exportAll().then(function(exportObj) {
for (var key in exportObj) {
baseObj[key] = [];
var times = exportObj[key];
for (var i = 0; i < times.length; i+= 1000) {
var timesSlice = LZString.compressToEncodedURIComponent(JSON.stringify(times.slice(i, i + 1000)));
var sliceHash = $.sha256(timesSlice);
slices['slice' + sliceHash] = timesSlice;
baseObj[key].push(sliceHash);
}
}, 'json').error(reject);
}
}).then(function() {
var tt = +new Date;
function uploadSlice(ids, datas) {
return new Promise(function(resolve, reject) {
$.post('https://cstimer.net/userdata2.php', {
'id': id,
'ids': ids.join(','),
'datas': datas.join(',')
}, function(val) {
if (val['retcode'] == 0) {
resolve(val);
} else {
reject(val);
}
}, 'json').error(reject);
});
}
var ret = Promise.resolve();
var ids = [id];
var datas = [LZString.compressToEncodedURIComponent(JSON.stringify(baseObj))];
//TODO check redaudant to reduce upload size
for (var key in slices) {
ids.push(key);
datas.push(slices[key]);
}
return ret.then(uploadSlice.bind(null, ids, datas));
});
}

Expand All @@ -180,7 +208,7 @@ var exportFunc = execMain(function() {
var msgfmt = EXPORT_WHICH_ITEM;
var msgf = [msg];
for (var ff = 0; ff < files.length; ff++) {
msgf.push((ff + 1) + '. ' + msgfmt.replace('%s', files[ff].size)
msgf.push((ff + 1) + '. ' + msgfmt.replace('%s', ~~files[ff].size || 'N/A')
.replace('%t', new Date(files[ff].modifiedTime).toLocaleString()));
}
return ~~prompt(msgf.join('\n'), '1');
Expand Down Expand Up @@ -217,30 +245,79 @@ var exportFunc = execMain(function() {
}
}
target.html('Import Data...');
$.post('https://cstimer.net/userdata.php', {
$.post('https://cstimer.net/userdata2.php', {
'id': id,
'offset': idx - 1
}, dataCallback, 'json').error(onerr).always(revert);
};

var dataCallback = function(val) {
var retcode = val['retcode'];
if (retcode == 0) {
try {
loadData(JSON.parse(LZString.decompressFromEncodedURIComponent(val['data'])));
} catch (err) {
alert(EXPORT_ERROR);
}
} else if (retcode == 404) {
if (retcode == 404) {
alert(EXPORT_NODATA);
} else {
return revert();
} else if (retcode != 0) {
alert(EXPORT_ERROR);
return revert();
}
var baseObj = {};
var ret = Promise.resolve();
function downloadSlice(keys, arrs) {
return new Promise(function(resolve, reject) {
$.post('https://cstimer.net/userdata2.php', {
'id': id,
'ids': keys.join(',')
}, function(val) {
if (val['retcode'] == 0) {
resolve(val);
} else {
reject(val);
}
}, 'json').error(reject);
}).then(function(val) {
var datas = [];
for (var key in val['datas']) {
datas[keys.indexOf(key)] = val['datas'][key];
};
for (var i = 0; i < keys.length; i++) {
if (datas[i] == undefined) {
console.log('error, incorrect data');
return Promise.reject();
}
Array.prototype.push.apply(arrs[i], JSON.parse(LZString.decompressFromEncodedURIComponent(datas[i])));
}
});
}
try {
var metaObj = JSON.parse(LZString.decompressFromEncodedURIComponent(val['data']));
//TODO compare with local data to skip some of downloads
var keys = [];
var arrs = [];
for (var key in metaObj) {
if (!key.startsWith('session') || metaObj[key].length == 0 || $.isArray(metaObj[key][0])) {
baseObj[key] = metaObj[key];
continue;
}
var times = [];
baseObj[key] = times;
for (var i = 0; i < metaObj[key].length; i++) {
keys.push('slice' + metaObj[key][i]);
arrs.push(times);
}
}
if (keys.length > 0) {
ret = ret.then(downloadSlice.bind(null, keys, arrs));
}
ret.then(loadData.bind(null, baseObj));
} catch (err) {
DEBUG && console.log('[export] error', err);
alert(EXPORT_ERROR);
revert();
}
revert();
};

if (kernel.getProp('expp')) {
$.post('https://cstimer.net/userdata.php', {
$.post('https://cstimer.net/userdata2.php', {
'id': id,
'cnt': 1
}, cntCallback, 'json').error(onerr).always(revert);
Expand Down

0 comments on commit 561b9d9

Please sign in to comment.