Skip to content

Commit 39bd0b8

Browse files
dpgeorgedpgeorge
committed
webassembly: Add JavaScript proxying, and js and jsffi modules.
This commit improves the webassembly port by adding: - Proxying of Python objects to JavaScript with a PyProxy type that lives on the JavaScript side. PyProxy implements JavaScript Proxy traps such as has, get, set and ownKeys, to make Python objects have functionality on the JavaScript side. - Proxying of JavaScript objects to Python with a JsProxy type that lives on the Python side. JsProxy passes through calls, attributes, subscription and iteration from Python to JavaScript. - A top-level API on the JavaScript side to construct a MicroPython interpreter instance via `loadMicroPython()`. That function returns an object that can be used to execute Python code, access the Python globals dict, access the Emscripten filesystem, and other things. This API is based on the API provided by Pyodide (https://pyodide.org/). As part of this, the top-level file is changed from `micropython.js` to `micropython.mjs`. - A Python `js` module which can be used to access all JavaScript-side symbols, for example the DOM when run within a browser. - A Python `jsffi` module with various helper functions like `create_proxy()` and `to_js()`. - A dedenting lexer which automatically dedents Python source code if every non-empty line in that source starts with a common whitespace prefix. This is very helpful when Python source code is indented within a string within HTML or JavaScript for formatting reasons. Signed-off-by: Damien George <[email protected]>
1 parent 691cd3a commit 39bd0b8

13 files changed

+1615
-9
lines changed

‎ports/webassembly/Makefile

+47-9
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,34 @@ CFLAGS += -std=c99 -Wall -Werror -Wdouble-promotion -Wfloat-conversion
4545
CFLAGS += -Os -DNDEBUG
4646
CFLAGS += $(INC)
4747

48+
EXPORTED_FUNCTIONS_EXTRA += ,\
49+
_mp_js_do_exec,\
50+
_mp_js_do_import,\
51+
_mp_js_register_js_module,\
52+
_proxy_c_init,\
53+
_proxy_c_to_js_call,\
54+
_proxy_c_to_js_delete_attr,\
55+
_proxy_c_to_js_dir,\
56+
_proxy_c_to_js_get_array,\
57+
_proxy_c_to_js_get_dict,\
58+
_proxy_c_to_js_get_type,\
59+
_proxy_c_to_js_has_attr,\
60+
_proxy_c_to_js_lookup_attr,\
61+
_proxy_c_to_js_store_attr,\
62+
_proxy_convert_mp_to_js_obj_cside
63+
64+
EXPORTED_RUNTIME_METHODS_EXTRA += ,\
65+
PATH,\
66+
PATH_FS,\
67+
UTF8ToString,\
68+
getValue,\
69+
lengthBytesUTF8,\
70+
setValue,\
71+
stringToUTF8
72+
4873
JSFLAGS += -s EXPORTED_FUNCTIONS="\
74+
_free,\
75+
_malloc,\
4976
_mp_js_init,\
5077
_mp_js_init_repl,\
5178
_mp_js_do_str,\
@@ -58,6 +85,7 @@ JSFLAGS += -s EXPORTED_RUNTIME_METHODS="\
5885
FS$(EXPORTED_RUNTIME_METHODS_EXTRA)"
5986
JSFLAGS += --js-library library.js
6087
JSFLAGS += -s SUPPORT_LONGJMP=emscripten
88+
JSFLAGS += -s MODULARIZE -s EXPORT_NAME=_createMicroPythonModule
6189

6290
################################################################################
6391
# Source files and libraries.
@@ -71,13 +99,21 @@ SRC_SHARED = $(addprefix shared/,\
7199
)
72100

73101
SRC_C += \
102+
lexer_dedent.c \
74103
main.c \
104+
modjs.c \
105+
modjsffi.c \
75106
mphalport.c \
107+
objjsproxy.c \
108+
proxy_c.c \
76109

77110
# List of sources for qstr extraction.
78111
SRC_QSTR += $(SRC_C) $(SRC_SHARED)
79112

80-
SRC_JS ?= wrapper.js
113+
SRC_JS += \
114+
api.js \
115+
objpyproxy.js \
116+
proxy_js.js \
81117

82118
OBJ += $(PY_O)
83119
OBJ += $(addprefix $(BUILD)/, $(SRC_SHARED:.c=.o))
@@ -86,23 +122,25 @@ OBJ += $(addprefix $(BUILD)/, $(SRC_C:.c=.o))
86122
################################################################################
87123
# Main targets.
88124

89-
.PHONY: all min test
125+
.PHONY: all min test test_min
90126

91-
all: $(BUILD)/micropython.js
127+
all: $(BUILD)/micropython.mjs
92128

93-
$(BUILD)/micropython.js: $(OBJ) library.js $(SRC_JS)
129+
$(BUILD)/micropython.mjs: $(OBJ) library.js $(SRC_JS)
94130
$(ECHO) "LINK $@"
95131
$(Q)emcc $(LDFLAGS) -o $@ $(OBJ) $(JSFLAGS)
96132
$(Q)cat $(SRC_JS) >> $@
97133

98-
$(BUILD)/micropython.min.js: $(BUILD)/micropython.js
134+
$(BUILD)/micropython.min.mjs: $(BUILD)/micropython.mjs
99135
$(TERSER) $< --compress --module -o $@
100136

101-
min: $(BUILD)/micropython.min.js
137+
min: $(BUILD)/micropython.min.mjs
138+
139+
test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py
140+
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
102141

103-
test: $(BUILD)/micropython.js $(TOP)/tests/run-tests.py
104-
$(eval DIRNAME=ports/$(notdir $(CURDIR)))
105-
cd $(TOP)/tests && MICROPY_MICROPYTHON=../ports/webassembly/node_run.sh ./run-tests.py -j1
142+
test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py
143+
cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
106144

107145
################################################################################
108146
# Remaining make rules.

‎ports/webassembly/api.js

+146
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/*
2+
* This file is part of the MicroPython project, http://micropython.org/
3+
*
4+
* The MIT License (MIT)
5+
*
6+
* Copyright (c) 2023-2024 Damien P. George
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* in the Software without restriction, including without limitation the rights
11+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
* copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be included in
16+
* all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24+
* THE SOFTWARE.
25+
*/
26+
27+
// Options:
28+
// - heapsize: size in bytes of the MicroPython GC heap.
29+
// - url: location to load `micropython.mjs`.
30+
// - stdin: function to return input characters.
31+
// - stdout: function that takes one argument, and is passed lines of stdout
32+
// output as they are produced. By default this is handled by Emscripten
33+
// and in a browser goes to console, in node goes to process.stdout.write.
34+
// - stderr: same behaviour as stdout but for error output.
35+
// - linebuffer: whether to buffer line-by-line to stdout/stderr.
36+
export async function loadMicroPython(options) {
37+
const { heapsize, url, stdin, stdout, stderr, linebuffer } = Object.assign(
38+
{ heapsize: 1024 * 1024, linebuffer: true },
39+
options,
40+
);
41+
const Module = {};
42+
Module.locateFile = (path, scriptDirectory) =>
43+
url || scriptDirectory + path;
44+
Module._textDecoder = new TextDecoder();
45+
if (stdin !== undefined) {
46+
Module.stdin = stdin;
47+
}
48+
if (stdout !== undefined) {
49+
if (linebuffer) {
50+
Module._stdoutBuffer = [];
51+
Module.stdout = (c) => {
52+
if (c === 10) {
53+
stdout(
54+
Module._textDecoder.decode(
55+
new Uint8Array(Module._stdoutBuffer),
56+
),
57+
);
58+
Module._stdoutBuffer = [];
59+
} else {
60+
Module._stdoutBuffer.push(c);
61+
}
62+
};
63+
} else {
64+
Module.stdout = (c) => stdout(new Uint8Array([c]));
65+
}
66+
}
67+
if (stderr !== undefined) {
68+
if (linebuffer) {
69+
Module._stderrBuffer = [];
70+
Module.stderr = (c) => {
71+
if (c === 10) {
72+
stderr(
73+
Module._textDecoder.decode(
74+
new Uint8Array(Module._stderrBuffer),
75+
),
76+
);
77+
Module._stderrBuffer = [];
78+
} else {
79+
Module._stderrBuffer.push(c);
80+
}
81+
};
82+
} else {
83+
Module.stderr = (c) => stderr(new Uint8Array([c]));
84+
}
85+
}
86+
const moduleLoaded = new Promise((r) => {
87+
Module.postRun = r;
88+
});
89+
_createMicroPythonModule(Module);
90+
await moduleLoaded;
91+
globalThis.Module = Module;
92+
proxy_js_init();
93+
const pyimport = (name) => {
94+
const value = Module._malloc(3 * 4);
95+
Module.ccall(
96+
"mp_js_do_import",
97+
"null",
98+
["string", "pointer"],
99+
[name, value],
100+
);
101+
return proxy_convert_mp_to_js_obj_jsside_with_free(value);
102+
};
103+
Module.ccall("mp_js_init", "null", ["number"], [heapsize]);
104+
Module.ccall("proxy_c_init", "null", [], []);
105+
return {
106+
_module: Module,
107+
PyProxy: PyProxy,
108+
FS: Module.FS,
109+
globals: {
110+
__dict__: pyimport("__main__").__dict__,
111+
get(key) {
112+
return this.__dict__[key];
113+
},
114+
set(key, value) {
115+
this.__dict__[key] = value;
116+
},
117+
delete(key) {
118+
delete this.__dict__[key];
119+
},
120+
},
121+
registerJsModule(name, module) {
122+
const value = Module._malloc(3 * 4);
123+
proxy_convert_js_to_mp_obj_jsside(module, value);
124+
Module.ccall(
125+
"mp_js_register_js_module",
126+
"null",
127+
["string", "pointer"],
128+
[name, value],
129+
);
130+
Module._free(value);
131+
},
132+
pyimport: pyimport,
133+
runPython(code) {
134+
const value = Module._malloc(3 * 4);
135+
Module.ccall(
136+
"mp_js_do_exec",
137+
"number",
138+
["string", "pointer"],
139+
[code, value],
140+
);
141+
return proxy_convert_mp_to_js_obj_jsside_with_free(value);
142+
},
143+
};
144+
}
145+
146+
globalThis.loadMicroPython = loadMicroPython;

‎ports/webassembly/lexer_dedent.c

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* This file is part of the MicroPython project, http://micropython.org/
3+
*
4+
* The MIT License (MIT)
5+
*
6+
* Copyright (c) 2023 Damien P. George
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* in the Software without restriction, including without limitation the rights
11+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
* copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be included in
16+
* all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24+
* THE SOFTWARE.
25+
*/
26+
27+
#include "lexer_dedent.h"
28+
29+
typedef struct _mp_reader_mem_dedent_t {
30+
size_t free_len; // if >0 mem is freed on close by: m_free(beg, free_len)
31+
const byte *beg;
32+
const byte *cur;
33+
const byte *end;
34+
size_t dedent_prefix;
35+
} mp_reader_mem_dedent_t;
36+
37+
// Work out the amount of common whitespace among all non-empty lines.
38+
static size_t dedent(const byte *text, size_t len) {
39+
size_t min_prefix = -1;
40+
size_t cur_prefix = 0;
41+
bool start_of_line = true;
42+
for (const byte *t = text; t < text + len; ++t) {
43+
if (*t == '\n') {
44+
start_of_line = true;
45+
cur_prefix = 0;
46+
} else if (start_of_line) {
47+
if (unichar_isspace(*t)) {
48+
++cur_prefix;
49+
} else {
50+
if (cur_prefix < min_prefix) {
51+
min_prefix = cur_prefix;
52+
if (min_prefix == 0) {
53+
return min_prefix;
54+
}
55+
}
56+
start_of_line = false;
57+
}
58+
}
59+
}
60+
return min_prefix;
61+
}
62+
63+
static mp_uint_t mp_reader_mem_dedent_readbyte(void *data) {
64+
mp_reader_mem_dedent_t *reader = (mp_reader_mem_dedent_t *)data;
65+
if (reader->cur < reader->end) {
66+
byte c = *reader->cur++;
67+
if (c == '\n') {
68+
for (size_t i = 0; i < reader->dedent_prefix; ++i) {
69+
if (*reader->cur == '\n') {
70+
break;
71+
}
72+
++reader->cur;
73+
}
74+
}
75+
return c;
76+
} else {
77+
return MP_READER_EOF;
78+
}
79+
}
80+
81+
static void mp_reader_mem_dedent_close(void *data) {
82+
mp_reader_mem_dedent_t *reader = (mp_reader_mem_dedent_t *)data;
83+
if (reader->free_len > 0) {
84+
m_del(char, (char *)reader->beg, reader->free_len);
85+
}
86+
m_del_obj(mp_reader_mem_dedent_t, reader);
87+
}
88+
89+
static void mp_reader_new_mem_dedent(mp_reader_t *reader, const byte *buf, size_t len, size_t free_len) {
90+
mp_reader_mem_dedent_t *rm = m_new_obj(mp_reader_mem_dedent_t);
91+
rm->free_len = free_len;
92+
rm->beg = buf;
93+
rm->cur = buf;
94+
rm->end = buf + len;
95+
rm->dedent_prefix = dedent(buf, len);
96+
reader->data = rm;
97+
reader->readbyte = mp_reader_mem_dedent_readbyte;
98+
reader->close = mp_reader_mem_dedent_close;
99+
}
100+
101+
mp_lexer_t *mp_lexer_new_from_str_len_dedent(qstr src_name, const char *str, size_t len, size_t free_len) {
102+
mp_reader_t reader;
103+
mp_reader_new_mem_dedent(&reader, (const byte *)str, len, free_len);
104+
return mp_lexer_new(src_name, reader);
105+
}

‎ports/webassembly/lexer_dedent.h

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* This file is part of the MicroPython project, http://micropython.org/
3+
*
4+
* The MIT License (MIT)
5+
*
6+
* Copyright (c) 2023-2024 Damien P. George
7+
*
8+
* Permission is hereby granted, free of charge, to any person obtaining a copy
9+
* of this software and associated documentation files (the "Software"), to deal
10+
* in the Software without restriction, including without limitation the rights
11+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
* copies of the Software, and to permit persons to whom the Software is
13+
* furnished to do so, subject to the following conditions:
14+
*
15+
* The above copyright notice and this permission notice shall be included in
16+
* all copies or substantial portions of the Software.
17+
*
18+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24+
* THE SOFTWARE.
25+
*/
26+
#ifndef MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H
27+
#define MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H
28+
29+
#include "py/lexer.h"
30+
31+
// This function creates a new "dedenting lexer" which automatically dedents the input
32+
// source code if every non-empty line in that source starts with a common whitespace
33+
// prefix. It does this dedenting inplace as the memory is read.
34+
mp_lexer_t *mp_lexer_new_from_str_len_dedent(qstr src_name, const char *str, size_t len, size_t free_len);
35+
36+
#endif // MICROPY_INCLUDED_WEBASSEMBLY_LEXER_DEDENT_H

0 commit comments

Comments
 (0)