Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add user-friendly, promise-based SQL.Worker class (in 65 lines-of-code) and deprecate dist/worker.sql-xxx.js #378

Closed
wants to merge 4 commits into from
Closed

add user-friendly, promise-based SQL.Worker class (in 65 lines-of-code) and deprecate dist/worker.sql-xxx.js #378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c1bcdef
- make src/worker.js isomorphic (will load in web-worker, nodejs, and…
kaizhu256 Apr 1, 2020
d56b472
- update Makefile to include isomorphic src/worker.js in all javascri…
kaizhu256 Apr 1, 2020
f381590
- update src/api.js with new class SQL.Worker and promise-method SQL.…
Apr 1, 2020
4c8c3c8
- fix out-of-order tic() toc() in examples/GUI/gui.js
Apr 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,13 @@ debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
mv $@ out/tmp-raw.js
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
rm out/tmp-raw.js

dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
mv $@ out/tmp-raw.js
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
rm out/tmp-raw.js

.PHONY: optimized
Expand All @@ -95,13 +95,13 @@ optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_ASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
mv $@ out/tmp-raw.js
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
rm out/tmp-raw.js

dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) $(EMFLAGS_PRE_JS_FILES) -o $@
mv $@ out/tmp-raw.js
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js src/worker.js > $@
rm out/tmp-raw.js

dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE_API_FILES) $(EXPORTED_METHODS_JSON_FILES)
Expand All @@ -110,21 +110,22 @@ dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) $(SOURCE
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
rm out/tmp-raw.js

# alias dist/worker.sql-xxx.js to dist/sql-xxx.js for legacy-compatibility
# Web worker API
.PHONY: worker
worker: dist/worker.sql-asm.js dist/worker.sql-asm-debug.js dist/worker.sql-wasm.js dist/worker.sql-wasm-debug.js

dist/worker.sql-asm.js: dist/sql-asm.js src/worker.js
cat $^ > $@
dist/worker.sql-asm.js: dist/sql-asm.js
cp $^ $@

dist/worker.sql-asm-debug.js: dist/sql-asm-debug.js src/worker.js
cat $^ > $@
dist/worker.sql-asm-debug.js: dist/sql-asm-debug.js
cp $^ $@

dist/worker.sql-wasm.js: dist/sql-wasm.js src/worker.js
cat $^ > $@
dist/worker.sql-wasm.js: dist/sql-wasm.js
cp $^ $@

dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js src/worker.js
cat $^ > $@
dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js
cp $^ $@

# Building it this way gets us a wrapper that _knows_ it's in worker mode, which is nice.
# However, since we can't tell emcc that we don't need the wasm generated, and just want the wrapper, we have to pay to have the .wasm generated
Expand Down
44 changes: 22 additions & 22 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,32 +191,32 @@ See : https://github.com/sql-js/sql.js/blob/master/test/test_node_file.js
If you don't want to run CPU-intensive SQL queries in your main application thread,
you can use the *more limited* WebWorker API.

You will need to download [dist/worker.sql-wasm.js](dist/worker.sql-wasm.js) [dist/worker.sql-wasm.wasm](dist/worker.sql-wasm.wasm).
You will need to download [dist/sql-wasm.js](dist/sql-wasm.js) [dist/sql-wasm.wasm](dist/sql-wasm.wasm).

Example:
```html
<script src="/dist/sql-wasm.js"></script>
<script>
var worker = new Worker("/dist/worker.sql-wasm.js");
worker.onmessage = () => {
console.log("Database opened");
worker.onmessage = event => {
console.log(event.data); // The result of the query
};

worker.postMessage({
id: 2,
action: "exec",
sql: "SELECT age,name FROM test WHERE id=$id",
params: { "$id": 1 }
});
};

worker.onerror = e => console.log("Worker error: ", e);
worker.postMessage({
id:1,
action:"open",
buffer:buf, /*Optional. An ArrayBuffer representing an SQLite Database file*/
});
(async function () {
var SQL = await initSqlJs({ locateFile: filename => `/dist/${filename}` });
var worker = new SQL.Worker("/dist/sql-wasm.js");
try {
await worker.postMessage({
id:1,
action:"open",
buffer:buf, /*Optional. An ArrayBuffer representing an SQLite Database file*/
});
var data = await worker.postMessage({
id: 2,
action: "exec",
sql: "SELECT age,name FROM test WHERE id=$id",
params: { "$id": 1 }
});
console.log(data);
} catch (sqlError) {
console.error(sqlError);
}
}());
</script>
```

Expand Down
262 changes: 132 additions & 130 deletions examples/GUI/gui.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,141 +1,143 @@
var execBtn = document.getElementById("execute");
var outputElm = document.getElementById('output');
var errorElm = document.getElementById('error');
var commandsElm = document.getElementById('commands');
var dbFileElm = document.getElementById('dbfile');
var savedbElm = document.getElementById('savedb');
(async function () {
"use strict";
var execBtn = document.getElementById("execute");
var outputElm = document.getElementById('output');
var errorElm = document.getElementById('error');
var commandsElm = document.getElementById('commands');
var dbFileElm = document.getElementById('dbfile');
var savedbElm = document.getElementById('savedb');

// Start the worker in which sql.js will run
var worker = new Worker("../../dist/worker.sql-wasm.js");
worker.onerror = error;
// Start the worker in which sql.js will run
var SQL = await initSqlJs({
locateFile: function (file) {
return `../../dist/${file}`;
}
});
var worker = new SQL.Worker("../../dist/sql-wasm-debug.js");

// Open a database
worker.postMessage({ action: 'open' });
// Open a database
worker.postMessage({ action: 'open' });

// Connect to the HTML element we 'print' to
function print(text) {
outputElm.innerHTML = text.replace(/\n/g, '<br>');
}
function error(e) {
console.log(e);
errorElm.style.height = '2em';
errorElm.textContent = e.message;
}
// Connect to the HTML element we 'print' to
function print(text) {
outputElm.innerHTML = text.replace(/\n/g, '<br>');
}
function error(e) {
console.log(e);
errorElm.style.height = '2em';
errorElm.textContent = e.message;
}

function noerror() {
errorElm.style.height = '0';
}
function noerror() {
errorElm.style.height = '0';
}

// Run a command in the database
function execute(commands) {
tic();
worker.onmessage = function (event) {
var results = event.data.results;
toc("Executing SQL");
if (!results) {
error({message: event.data.error});
return;
}
// Run a command in the database
async function execute(commands) {
tic();
outputElm.textContent = "Fetching results...";
var data;
try {
data = await worker.postMessage({ action: 'exec', sql: commands });
} catch (errorCaught) {
error({message: errorCaught});
return;
}
var results = data.results;
toc("Executing SQL");
tic();
outputElm.innerHTML = "";
for (var i = 0; i < results.length; i++) {
outputElm.appendChild(tableCreate(results[i].columns, results[i].values));
}
toc("Displaying results");
}

tic();
outputElm.innerHTML = "";
for (var i = 0; i < results.length; i++) {
outputElm.appendChild(tableCreate(results[i].columns, results[i].values));
}
toc("Displaying results");
}
worker.postMessage({ action: 'exec', sql: commands });
outputElm.textContent = "Fetching results...";
}
// Create an HTML table
var tableCreate = function () {
function valconcat(vals, tagName) {
if (vals.length === 0) return '';
var open = '<' + tagName + '>', close = '</' + tagName + '>';
return open + vals.join(close + open) + close;
}
return function (columns, values) {
var tbl = document.createElement('table');
var html = '<thead>' + valconcat(columns, 'th') + '</thead>';
var rows = values.map(function (v) { return valconcat(v, 'td'); });
html += '<tbody>' + valconcat(rows, 'tr') + '</tbody>';
tbl.innerHTML = html;
return tbl;
}
}();

// Create an HTML table
var tableCreate = function () {
function valconcat(vals, tagName) {
if (vals.length === 0) return '';
var open = '<' + tagName + '>', close = '</' + tagName + '>';
return open + vals.join(close + open) + close;
}
return function (columns, values) {
var tbl = document.createElement('table');
var html = '<thead>' + valconcat(columns, 'th') + '</thead>';
var rows = values.map(function (v) { return valconcat(v, 'td'); });
html += '<tbody>' + valconcat(rows, 'tr') + '</tbody>';
tbl.innerHTML = html;
return tbl;
}
}();
// Execute the commands when the button is clicked
function execEditorContents() {
noerror()
execute(editor.getValue() + ';');
}
execBtn.addEventListener("click", execEditorContents, true);

// Execute the commands when the button is clicked
function execEditorContents() {
noerror()
execute(editor.getValue() + ';');
}
execBtn.addEventListener("click", execEditorContents, true);
// Performance measurement functions
var tictime;
if (!window.performance || !performance.now) { window.performance = { now: Date.now } }
function tic() { tictime = performance.now() }
function toc(msg) {
var dt = performance.now() - tictime;
console.log((msg || 'toc') + ": " + dt + "ms");
}

// Performance measurement functions
var tictime;
if (!window.performance || !performance.now) { window.performance = { now: Date.now } }
function tic() { tictime = performance.now() }
function toc(msg) {
var dt = performance.now() - tictime;
console.log((msg || 'toc') + ": " + dt + "ms");
}
// Add syntax highlihjting to the textarea
var editor = CodeMirror.fromTextArea(commandsElm, {
mode: 'text/x-mysql',
viewportMargin: Infinity,
indentWithTabs: true,
smartIndent: true,
lineNumbers: true,
matchBrackets: true,
autofocus: true,
extraKeys: {
"Ctrl-Enter": execEditorContents,
"Ctrl-S": savedb,
}
});

// Add syntax highlihjting to the textarea
var editor = CodeMirror.fromTextArea(commandsElm, {
mode: 'text/x-mysql',
viewportMargin: Infinity,
indentWithTabs: true,
smartIndent: true,
lineNumbers: true,
matchBrackets: true,
autofocus: true,
extraKeys: {
"Ctrl-Enter": execEditorContents,
"Ctrl-S": savedb,
}
});
// Load a db from a file
dbFileElm.onchange = function () {
var f = dbFileElm.files[0];
var r = new FileReader();
r.onload = async function () {
tic();
try {
await worker.postMessage({ action: 'open', buffer: r.result }, [r.result]);
}
catch (exception) {
await worker.postMessage({ action: 'open', buffer: r.result });
}
toc("Loading database from file");
// Show the schema of the loaded database
editor.setValue("SELECT `name`, `sql`\n FROM `sqlite_master`\n WHERE type='table';");
execEditorContents();
}
r.readAsArrayBuffer(f);
}

// Load a db from a file
dbFileElm.onchange = function () {
var f = dbFileElm.files[0];
var r = new FileReader();
r.onload = function () {
worker.onmessage = function () {
toc("Loading database from file");
// Show the schema of the loaded database
editor.setValue("SELECT `name`, `sql`\n FROM `sqlite_master`\n WHERE type='table';");
execEditorContents();
};
tic();
try {
worker.postMessage({ action: 'open', buffer: r.result }, [r.result]);
}
catch (exception) {
worker.postMessage({ action: 'open', buffer: r.result });
}
}
r.readAsArrayBuffer(f);
}

// Save the db to a file
function savedb() {
worker.onmessage = function (event) {
toc("Exporting the database");
var arraybuff = event.data.buffer;
var blob = new Blob([arraybuff]);
var a = document.createElement("a");
document.body.appendChild(a);
a.href = window.URL.createObjectURL(blob);
a.download = "sql.db";
a.onclick = function () {
setTimeout(function () {
window.URL.revokeObjectURL(a.href);
}, 1500);
};
a.click();
};
tic();
worker.postMessage({ action: 'export' });
}
savedbElm.addEventListener("click", savedb, true);
// Save the db to a file
async function savedb() {
tic();
var data = await worker.postMessage({ action: 'export' });
toc("Exporting the database");
var arraybuff = data.buffer;
var blob = new Blob([arraybuff]);
var a = document.createElement("a");
document.body.appendChild(a);
a.href = window.URL.createObjectURL(blob);
a.download = "sql.db";
a.onclick = function () {
setTimeout(function () {
window.URL.revokeObjectURL(a.href);
}, 1500);
};
a.click();
}
savedbElm.addEventListener("click", savedb, true);
}());
Loading