Skip to content

Commit 5dd2220

Browse files
lovasoalovasoa
authored
Merge pull request #255 from Taytay/wasm
Compile sql.js to wasm instead of "just" asm.js
2 parents 9e69650 + 15eea72 commit 5dd2220

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+899906
-584201
lines changed

‎.gitignore

+4-9
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,7 @@
11
node_modules/
22
*~
33

4-
# LLVM bitcode
5-
c/sqlite3.bc
6-
c/extension-functions.bc
7-
# Intermediary js files
8-
js/api.js
9-
js/sql-noopt-raw.js.map
10-
js/sql-optimized.js
11-
js/worker.js
12-
js/sql-raw.js.map
4+
# Intermediary files:
5+
out/
6+
sqlite-src/
7+
cache/

‎AUTHORS

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ Ophir LOJKINE <[email protected]> (https://github.com/lovasoa)
22
@kripken
33
@hankinsoft
44
@firien
5-
@dinedal
5+
@dinedal
6+
@taytay

‎GUI/index.html

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<html>
2+
<body>
3+
<!-- Add a link to the new page because there are likely links pointing here from elsewhere -->
4+
This page has moved to: <a href="../examples/GUI/index.html">../examples/GUI/index.html</a>
5+
</body>
6+
</html>

‎LICENSE

+22
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,25 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
2020
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
2121
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2222
SOFTWARE.
23+
24+
25+
26+
# Some portions of the Makefile taken from:
27+
Copyright 2017 Ryusei Yamaguchi
28+
29+
Permission is hereby granted, free of charge, to any person obtaining a copy of
30+
this software and associated documentation files (the "Software"), to deal in
31+
the Software without restriction, including without limitation the rights to
32+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
33+
the Software, and to permit persons to whom the Software is furnished to do so,
34+
subject to the following conditions:
35+
36+
The above copyright notice and this permission notice shall be included in all
37+
copies or substantial portions of the Software.
38+
39+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
40+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
41+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
42+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
43+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
44+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

‎Makefile

+167-30
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,187 @@
1-
EMCC=emcc
1+
# Note: Last built with version 1.38.30 of Emscripten
22

3-
CFLAGS=-O2 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DISABLE_LFS -DLONGDOUBLE_TYPE=double -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS
3+
# TODO: Emit a file showing which version of emcc and SQLite was used to compile the emitted output.
4+
# TODO: Create a release on Github with these compiled assets rather than checking them in
5+
# TODO: Consider creating different files based on browser vs module usage: https://github.com/vuejs/vue/tree/dev/dist
46

5-
all: js/sql.js debug js/worker.sql.js memory-growth
7+
# I got this handy makefile syntax from : https://github.com/mandel59/sqlite-wasm (MIT License) Credited in LICENSE
8+
# To use another version of Sqlite, visit https://www.sqlite.org/download.html and copy the appropriate values here:
9+
SQLITE_AMALGAMATION = sqlite-amalgamation-3280000
10+
SQLITE_AMALGAMATION_ZIP_URL = https://www.sqlite.org/2019/sqlite-amalgamation-3280000.zip
11+
SQLITE_AMALGAMATION_ZIP_SHA1 = eb82fcc95104c8e2d9550ab023c1054b9cc40a76
612

7-
# RESERVED_FUNCTION_POINTERS setting is used for registering custom functions
8-
optimized: EMFLAGS= --memory-init-file 0 -O3 -s INLINING_LIMIT=50 -s RESERVED_FUNCTION_POINTERS=64
9-
optimized: js/sql-optimized.js
13+
# Note that extension-functions.c hasn't been updated since 2010-02-06, so likely doesn't need to be updated
14+
EXTENSION_FUNCTIONS = extension-functions.c
15+
EXTENSION_FUNCTIONS_URL = https://www.sqlite.org/contrib/download/extension-functions.c?get=25
16+
EXTENSION_FUNCTIONS_SHA1 = c68fa706d6d9ff98608044c00212473f9c14892f
1017

11-
memory-growth: EMFLAGS= --memory-init-file 0 -O3 -s INLINING_LIMIT=50 -s RESERVED_FUNCTION_POINTERS=64 -s ALLOW_MEMORY_GROWTH=1
12-
memory-growth: js/sql-memory-growth.js
18+
EMCC=emcc
1319

14-
debug: EMFLAGS= -O1 -g -s INLINING_LIMIT=10 -s RESERVED_FUNCTION_POINTERS=64
15-
debug: js/sql-debug.js
20+
CFLAGS=-O2 -DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_DISABLE_LFS -DLONGDOUBLE_TYPE=double -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_FTS3_PARENTHESIS
1621

17-
js/sql.js: optimized
18-
cp js/sql-optimized.js js/sql.js
22+
# When compiling to WASM, enabling memory-growth is not expected to make much of an impact, so we enable it for all builds
23+
# Since tihs is a library and not a standalone executable, we don't want to catch unhandled Node process exceptions
24+
# So, we do : `NODEJS_CATCH_EXIT=0`, which fixes issue: https://github.com/kripken/sql.js/issues/173 and https://github.com/kripken/sql.js/issues/262
25+
EMFLAGS = \
26+
--memory-init-file 0 \
27+
-s RESERVED_FUNCTION_POINTERS=64 \
28+
-s EXPORTED_FUNCTIONS=@src/exported_functions.json \
29+
-s EXTRA_EXPORTED_RUNTIME_METHODS=@src/exported_runtime_methods.json \
30+
-s SINGLE_FILE=0 \
31+
-s NODEJS_CATCH_EXIT=0
32+
33+
EMFLAGS_WASM = \
34+
-s WASM=1 \
35+
-s ALLOW_MEMORY_GROWTH=1
36+
37+
EMFLAGS_OPTIMIZED= \
38+
-s INLINING_LIMIT=50 \
39+
-O3 \
40+
--closure 1
41+
42+
EMFLAGS_DEBUG = \
43+
-s INLINING_LIMIT=10 \
44+
-O1
45+
46+
BITCODE_FILES = out/sqlite3.bc out/extension-functions.bc
47+
OUTPUT_WRAPPER_FILES = src/shell-pre.js src/shell-post.js
48+
49+
all: optimized debug worker
50+
51+
.PHONY: debug
52+
debug: dist/sql-asm-debug.js dist/sql-wasm-debug.js
53+
54+
dist/sql-asm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js src/exported_functions.json src/exported_runtime_methods.json
55+
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) -s WASM=0 $(BITCODE_FILES) --pre-js out/api.js -o $@
56+
mv $@ out/tmp-raw.js
57+
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
58+
rm out/tmp-raw.js
59+
60+
dist/sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js src/exported_functions.json src/exported_runtime_methods.json
61+
$(EMCC) $(EMFLAGS) $(EMFLAGS_DEBUG) $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js out/api.js -o $@
62+
mv $@ out/tmp-raw.js
63+
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
64+
rm out/tmp-raw.js
65+
66+
.PHONY: optimized
67+
optimized: dist/sql-asm.js dist/sql-wasm.js dist/sql-asm-memory-growth.js
68+
69+
dist/sql-asm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js src/exported_functions.json src/exported_runtime_methods.json
70+
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) -s WASM=0 $(BITCODE_FILES) --pre-js out/api.js -o $@
71+
mv $@ out/tmp-raw.js
72+
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
73+
rm out/tmp-raw.js
74+
75+
dist/sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js src/exported_functions.json src/exported_runtime_methods.json
76+
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js out/api.js -o $@
77+
mv $@ out/tmp-raw.js
78+
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
79+
rm out/tmp-raw.js
80+
81+
dist/sql-asm-memory-growth.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js src/exported_functions.json src/exported_runtime_methods.json
82+
$(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) -s WASM=0 -s ALLOW_MEMORY_GROWTH=1 $(BITCODE_FILES) --pre-js out/api.js -o $@
83+
mv $@ out/tmp-raw.js
84+
cat src/shell-pre.js out/tmp-raw.js src/shell-post.js > $@
85+
rm out/tmp-raw.js
1986

20-
js/sql%.js: js/shell-pre.js js/sql%-raw.js js/shell-post.js
21-
cat $^ > $@
2287

23-
js/sql%-raw.js: c/sqlite3.bc c/extension-functions.bc js/api.js exported_functions
24-
$(EMCC) $(EMFLAGS) -s EXPORTED_FUNCTIONS=@exported_functions -s EXTRA_EXPORTED_RUNTIME_METHODS=@exported_runtime_methods c/extension-functions.bc c/sqlite3.bc --post-js js/api.js -o $@ ;\
88+
# Web worker API
89+
.PHONY: worker
90+
worker: dist/worker.sql-asm.js dist/worker.sql-asm-debug.js dist/worker.sql-wasm.js dist/worker.sql-wasm-debug.js
2591

26-
js/api.js: coffee/api.coffee coffee/exports.coffee coffee/api-data.coffee
92+
out/worker.js: src/worker.coffee
2793
cat $^ | coffee --bare --compile --stdio > $@
2894

29-
# Web worker API
30-
worker: js/worker.sql.js
31-
js/worker.js: coffee/worker.coffee
32-
cat $^ | coffee --bare --compile --stdio > $@
95+
dist/worker.sql-asm.js: dist/sql-asm.js out/worker.js
96+
cat $^ > $@
3397

34-
js/worker.sql.js: js/sql.js js/worker.js
98+
dist/worker.sql-asm-debug.js: dist/sql-asm-debug.js out/worker.js
3599
cat $^ > $@
36100

37-
c/sqlite3.bc: c/sqlite3.c
101+
dist/worker.sql-wasm.js: dist/sql-wasm.js out/worker.js
102+
cat $^ > $@
103+
104+
dist/worker.sql-wasm-debug.js: dist/sql-wasm-debug.js out/worker.js
105+
cat $^ > $@
106+
107+
# Building it this way gets us a wrapper that _knows_ it's in worker mode, which is nice.
108+
# However, since we can't tell emcc that we don't need the wasm generated, and just want the wrapper, we have to pay to have the .wasm generated
109+
# even though we would have already generated it with our sql-wasm.js target above.
110+
# This would be made easier if this is implemented: https://github.com/emscripten-core/emscripten/issues/8506
111+
# dist/worker.sql-wasm.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js out/worker.js src/exported_functions.json src/exported_runtime_methods.json dist/sql-wasm-debug.wasm
112+
# $(EMCC) $(EMFLAGS) $(EMFLAGS_OPTIMIZED) -s ENVIRONMENT=worker -s $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js out/api.js -o out/sql-wasm.js
113+
# mv out/sql-wasm.js out/tmp-raw.js
114+
# cat src/shell-pre.js out/tmp-raw.js src/shell-post.js out/worker.js > $@
115+
# #mv out/sql-wasm.wasm dist/sql-wasm.wasm
116+
# rm out/tmp-raw.js
117+
118+
# dist/worker.sql-wasm-debug.js: $(BITCODE_FILES) $(OUTPUT_WRAPPER_FILES) out/api.js out/worker.js src/exported_functions.json src/exported_runtime_methods.json dist/sql-wasm-debug.wasm
119+
# $(EMCC) -s ENVIRONMENT=worker $(EMFLAGS) $(EMFLAGS_DEBUG) -s ENVIRONMENT=worker -s WASM_BINARY_FILE=sql-wasm-foo.debug $(EMFLAGS_WASM) $(BITCODE_FILES) --pre-js out/api.js -o out/sql-wasm-debug.js
120+
# mv out/sql-wasm-debug.js out/tmp-raw.js
121+
# cat src/shell-pre.js out/tmp-raw.js src/shell-post.js out/worker.js > $@
122+
# #mv out/sql-wasm-debug.wasm dist/sql-wasm-debug.wasm
123+
# rm out/tmp-raw.js
124+
125+
out/api.js: src/output-pre.js src/api.coffee src/exports.coffee src/api-data.coffee src/output-post.js
126+
cat src/api.coffee src/exports.coffee src/api-data.coffee | coffee --bare --compile --stdio > $@
127+
cat src/output-pre.js $@ src/output-post.js > out/api-wrapped.js
128+
mv out/api-wrapped.js $@
129+
130+
out/sqlite3.bc: sqlite-src/$(SQLITE_AMALGAMATION)
38131
# Generate llvm bitcode
39-
$(EMCC) $(CFLAGS) c/sqlite3.c -o c/sqlite3.bc
132+
$(EMCC) $(CFLAGS) sqlite-src/$(SQLITE_AMALGAMATION)/sqlite3.c -o $@
133+
134+
out/extension-functions.bc: sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS)
135+
$(EMCC) $(CFLAGS) -s LINKABLE=1 sqlite-src/$(SQLITE_AMALGAMATION)/extension-functions.c -o $@
136+
137+
# TODO: This target appears to be unused. If we re-instatate it, we'll need to add more files inside of the JS folder
138+
# module.tar.gz: test package.json AUTHORS README.md dist/sql-asm.js
139+
# tar --create --gzip $^ > $@
140+
141+
## cache
142+
143+
.PHONY: clean-cache
144+
clean-cache:
145+
rm -rf cache
146+
147+
cache/$(SQLITE_AMALGAMATION).zip:
148+
mkdir -p cache
149+
curl -LsSf '$(SQLITE_AMALGAMATION_ZIP_URL)' -o $@
150+
151+
cache/$(EXTENSION_FUNCTIONS):
152+
mkdir -p cache
153+
curl -LsSf '$(EXTENSION_FUNCTIONS_URL)' -o $@
154+
155+
## sqlite-src
156+
157+
.PHONY: clean-sqlite-src
158+
clean-sqlite-src:
159+
rm -rf sqlite
160+
161+
.PHONY: sqlite-src
162+
sqlite-src: sqlite-src/$(SQLITE_AMALGAMATION) sqlite-src/$(EXTENSION_FUNCTIONS)
163+
164+
sqlite-src/$(SQLITE_AMALGAMATION): cache/$(SQLITE_AMALGAMATION).zip
165+
mkdir -p sqlite-src
166+
echo '$(SQLITE_AMALGAMATION_ZIP_SHA1) ./cache/$(SQLITE_AMALGAMATION).zip' > cache/check.txt
167+
sha1sum -c cache/check.txt
168+
rm -rf $@
169+
unzip 'cache/$(SQLITE_AMALGAMATION).zip' -d sqlite-src/
170+
touch $@
40171

41-
c/extension-functions.bc: c/extension-functions.c
42-
$(EMCC) $(CFLAGS) -s LINKABLE=1 c/extension-functions.c -o c/extension-functions.bc
172+
sqlite-src/$(SQLITE_AMALGAMATION)/$(EXTENSION_FUNCTIONS): cache/$(EXTENSION_FUNCTIONS)
173+
mkdir -p sqlite-src
174+
echo '$(EXTENSION_FUNCTIONS_SHA1) ./cache/$(EXTENSION_FUNCTIONS)' > cache/check.txt
175+
sha1sum -c cache/check.txt
176+
cp 'cache/$(EXTENSION_FUNCTIONS)' $@
43177

44-
module.tar.gz: test package.json AUTHORS README.md js/sql.js
45-
tar --create --gzip $^ > $@
46178

47-
clean:
48-
rm -rf js/sql.js js/api.js js/sql*-raw.js js/worker.sql.js js/worker.js js/sql-memory-growth.js c/sqlite3.bc c/extension-functions.bc
179+
.PHONY: clean
180+
clean:
181+
rm -rf out/* dist/*
49182

183+
.PHONY: clean-all
184+
clean-all:
185+
rm -f out/* dist/* cache/*
186+
rm -rf sqlite-src/
50187

0 commit comments

Comments
 (0)