Skip to content

Commit

Permalink
2.2.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@DerThorsten
DerThorsten committed Aug 1, 2024
1 parent c12cb4c commit 18d7291
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 106 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.9)

project(pyjs VERSION 2.1.0 DESCRIPTION "pyjs")
project(pyjs VERSION 2.1.1 DESCRIPTION "pyjs")

option(BUILD_RUNTIME_BROWSER "Build runtime" ON)
option(WITH_NODE_TESTS "With node tests" OFF)
Expand Down
14 changes: 11 additions & 3 deletions include/pyjs/pre_js/init.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ Module._is_initialized = false



Module['init_phase_1'] = async function(prefix, python_version) {
Module['init_phase_1'] = async function(prefix, python_version, verbose) {

if(verbose){console.log("in init phase 1");}
let version_str = `${python_version[0]}.${python_version[1]}`;

// list of python objects we need to delete when cleaning up
Expand Down Expand Up @@ -133,12 +134,15 @@ Module['init_phase_1'] = async function(prefix, python_version) {
return ret['ret']
}
};
if(verbose){console.log("in init phase 2 done");}
}

Module['init_phase_2'] = function(prefix, python_version) {
Module['init_phase_2'] = function(prefix, python_version, verbose) {
let default_scope = Module["default_scope"];

// make the python pyjs module easy available
if(verbose){console.log("in init phase 2");}

Module.exec(`
import traceback
try:
Expand All @@ -149,24 +153,27 @@ except Exception as e:
raise e
`)

if(verbose){console.log("assign pyobjects I");}
Module.py_pyjs = Module.eval("pyjs")
Module._py_objects.push(Module.py_pyjs);


// execute a script and return the value of the last expression
if(verbose){console.log("assign pyobjects II");}
Module._py_exec_eval = Module.eval("pyjs.exec_eval")
Module._py_objects.push(Module._py_exec_eval)
Module.exec_eval = function(script, globals=default_scope, locals=default_scope){
return Module._py_exec_eval.py_call(script, globals, locals)
}

// ansync execute a script and return the value of the last expression
if(verbose){console.log("assign pyobjects III");}
Module._py_async_exec_eval = Module.eval("pyjs.async_exec_eval")
Module._py_objects.push(Module._py_async_exec_eval)
Module.async_exec_eval = async function(script, globals=default_scope, locals=default_scope){
return await Module._py_async_exec_eval.py_call(script, globals, locals)
}

if(verbose){console.log("assign pyobjects IV");}
Module._add_resolve_done_callback = Module.exec_eval(`
import asyncio
def _add_resolve_done_callback(future, resolve, reject):
Expand Down Expand Up @@ -240,4 +247,5 @@ def _mock_webbrowser():
_mock_webbrowser()
del _mock_webbrowser
`);
if(verbose){console.log("init phase 2 done");}
}
237 changes: 135 additions & 102 deletions include/pyjs/pre_js/load_pkg.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ Module["mkdirs"] = function (dirname) {
}


function untar_from_python(tarball_path, target_dir = "") {

Module["_untar_from_python"] = function(tarball_path, target_dir = "") {
Module.exec(`
def _py_untar(tarball_path, target_dir):
import tarfile
Expand Down Expand Up @@ -65,130 +66,162 @@ def _py_untar(tarball_path, target_dir):
return JSON.parse(shared_libs)
}

Module["_untar_from_python"] = untar_from_python

async function bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose = false) {
// fetch python package
let python_package_url = `${package_tarballs_root_url}/${python_package.filename}`

if (verbose) {
console.log(`fetching python package from ${python_package_url}`)
}
let byte_array = await fetchByteArray(python_package_url)

const python_tarball_path = `/package_tarballs/${python_package.filename}`;
if(verbose){
console.log(`extract ${python_tarball_path} (${byte_array.length} bytes)`)
}
Module.FS.writeFile(python_tarball_path, byte_array);
Module._untar(python_tarball_path, prefix);


// split version string into major and minor and patch version
let version = python_package.version.split(".").map(x => parseInt(x));



await Module.init_phase_1(prefix, version);
}


Module["bootstrap_from_empack_packed_environment"] = async function
(
packages_json_url,
package_tarballs_root_url,
verbose = false,
verbose = true,
skip_loading_shared_libs = false
) {

function splitPackages(packages) {
// find package with name "python" and remove it from the list
let python_package = undefined
for (let i = 0; i < packages.length; i++) {
if (packages[i].name == "python") {
python_package = packages[i]
packages.splice(i, 1)
break
)
{
try{

verbose=true;

function splitPackages(packages) {
// find package with name "python" and remove it from the list
let python_package = undefined
for (let i = 0; i < packages.length; i++) {
if (packages[i].name == "python") {
python_package = packages[i]
packages.splice(i, 1)
break
}
}
if (python_package == undefined) {
throw new Error("no python package found in package.json")
}
return { python_package, packages }
}
if (python_package == undefined) {
throw new Error("no python package found in package.json")
}
return { python_package, packages }
}


function makeDirs() {
if (!Module.FS.isDir(`/package_tarballs`)) {
Module.FS.mkdir(`/package_tarballs`);



async function fetchAndUntar
(
package_tarballs_root_url,
python_is_ready_promise,
pkg,
verbose
) {
let package_url = `${package_tarballs_root_url}/${pkg.filename}`
if (verbose) {
console.log(`!!fetching pkg ${pkg.name} from ${package_url}`)
}
let byte_array = await fetchByteArray(package_url)
const tarball_path = `/package_tarballs/${pkg.filename}`;
Module.FS.writeFile(tarball_path, byte_array);
if(verbose){
console.log(`!!extract ${tarball_path} (${byte_array.length} bytes)`)
}
if(verbose){console.log("await python_is_ready_promise");}
await python_is_ready_promise;
return Module["_untar_from_python"](tarball_path);
}
}


async function fetchAndUntar
(
package_tarballs_root_url,
python_is_ready_promise,
pkg,
verbose = false
) {
let package_url = `${package_tarballs_root_url}/${pkg.filename}`
if (verbose) {
console.log(`fetching pkg ${pkg.name} from ${package_url}`)


async function bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose) {
// fetch python package
let python_package_url = `${package_tarballs_root_url}/${python_package.filename}`

if (verbose) {
console.log(`fetching python package from ${python_package_url}`)
}
let byte_array = await fetchByteArray(python_package_url)

const python_tarball_path = `/package_tarballs/${python_package.filename}`;
if(verbose){
console.log(`extract ${python_tarball_path} (${byte_array.length} bytes)`)
}
Module.FS.writeFile(python_tarball_path, byte_array);
if(verbose){console.log("untar_from_python");}
Module._untar(python_tarball_path, prefix);





// split version string into major and minor and patch version
let version = python_package.version.split(".").map(x => parseInt(x));


if(verbose){console.log("start init_phase_1");}
await Module.init_phase_1(prefix, version, verbose);
}
let byte_array = await fetchByteArray(package_url)
const tarball_path = `/package_tarballs/${pkg.filename}`;
Module.FS.writeFile(tarball_path, byte_array);



if(verbose){
console.log(`extract ${tarball_path} (${byte_array.length} bytes)`)
console.log("fetching packages.json from", packages_json_url)
}
await python_is_ready_promise;
return untar_from_python(tarball_path);
}






// fetch json with list of all packages
let empack_env_meta = await fetchJson(packages_json_url);
let all_packages = empack_env_meta.packages;
let prefix = empack_env_meta.prefix;
makeDirs();
// fetch json with list of all packages
let empack_env_meta = await fetchJson(packages_json_url);
let all_packages = empack_env_meta.packages;
let prefix = empack_env_meta.prefix;

// enusre there is python and split it from the rest
let splitted = splitPackages(all_packages);
let packages = splitted.packages;
let python_package = splitted.python_package;
let python_version = python_package.version.split(".").map(x => parseInt(x));

// fetch init python itself
let python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package);

// create array with size
let shared_libs = await Promise.all(packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)));

Module.init_phase_2(prefix, python_version);
if(verbose){
console.log("makeDirs");
}
Module.create_directories("/package_tarballs");

// enusre there is python and split it from the rest
if(verbose){console.log("splitPackages");}
let splitted = splitPackages(all_packages);
let packages = splitted.packages;
let python_package = splitted.python_package;
let python_version = python_package.version.split(".").map(x => parseInt(x));

// fetch init python itself
console.log("--bootstrap_python");
if(verbose){
console.log("bootstrap_python");
}
let python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose);

if(!skip_loading_shared_libs){
// instantiate all packages
for (let i = 0; i < packages.length; i++) {
// create array with size
if(verbose){
console.log("fetchAndUntarAll");
}
let shared_libs = await Promise.all(packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)));

// if we have any shared libraries, load them
if (shared_libs[i].length > 0) {
if(verbose){
console.log("init_phase_2");
}
Module.init_phase_2(prefix, python_version, verbose);

for (let j = 0; j < shared_libs[i].length; j++) {
let sl = shared_libs[i][j];
if(verbose){
console.log("init shared");
}
if(!skip_loading_shared_libs){
// instantiate all packages
for (let i = 0; i < packages.length; i++) {

// if we have any shared libraries, load them
if (shared_libs[i].length > 0) {

for (let j = 0; j < shared_libs[i].length; j++) {
let sl = shared_libs[i][j];
}
await Module._loadDynlibsFromPackage(
prefix,
python_version,
packages[i].name,
false,
shared_libs[i]
)
}
await Module._loadDynlibsFromPackage(
prefix,
python_version,
packages[i].name,
false,
shared_libs[i]
)
}
}
if(verbose){
console.log("done bootstrapping");}
}
catch(e){
console.log("error in bootstrapping process")
console.error(e);
}
}

0 comments on commit 18d7291

Please sign in to comment.