Skip to content

Commit

Permalink
Bootstrap (emscripten-forge#43)
Browse files Browse the repository at this point in the history 
added dynamic loading
  • Loading branch information
@DerThorsten
DerThorsten authored Apr 14, 2023
1 parent 47f515f commit 5ef613d
Show file tree
Hide file tree
Showing 8 changed files with 258 additions and 232 deletions.
105 changes: 14 additions & 91 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ jobs:
run: |
micromamba activate pyjs-wasm
python -m pip install git+https://github.com/emscripten-forge/empack@main --no-deps --ignore-installed
python -m pip install git+https://github.com/emscripten-forge/empack@rootpack --no-deps --ignore-installed
python -m pip install git+https://github.com/DerThorsten/pyjs-code-runner@relocate_env --no-deps --ignore-installed
emsdk activate ${{matrix.emsdk_ver}}
source $CONDA_EMSDK_DIR/emsdk_env.sh
Expand Down Expand Up @@ -86,6 +86,15 @@ jobs:
popd
- name: setup env with numpy
run: |
micromamba activate pyjs-wasm
micromamba create -n pyjs-build-wasm \
--platform=emscripten-32 \
-c https://repo.mamba.pm/emscripten-forge \
-c https://repo.mamba.pm/conda-forge \
--yes \
python pytest numpy
- name: Test in browser-main
Expand All @@ -94,7 +103,7 @@ jobs:
pyjs_code_runner run script \
browser-main \
--conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm \
--mount $(pwd)/tests:tests \
--mount $(pwd)/tests:/tests \
--script main.py \
--work-dir /tests \
--pyjs-dir $(pwd)/build \
Expand All @@ -109,7 +118,7 @@ jobs:
pyjs_code_runner run script \
browser-worker \
--conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm \
--mount $(pwd)/tests:tests \
--mount $(pwd)/tests:/tests \
--script main.py \
--work-dir /tests \
--pyjs-dir $(pwd)/build \
Expand All @@ -134,96 +143,10 @@ jobs:
pyjs_code_runner run script \
browser-main \
--conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm \
--mount $(pwd)/tests:tests \
--mount $(pwd)/tests:/tests \
--script main.py \
--work-dir /tests \
--pyjs-dir $(pwd)/build \
--headless \
--async-main \
--no-cache
test-node:

runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
emsdk_ver: ["3.1.27"]

steps:
- uses: actions/checkout@v2

- name: Get number of CPU cores
uses: SimenB/github-actions-cpu-cores@v1

- name: Install mamba
uses: mamba-org/provision-with-micromamba@main
with:
environment-file: environment-dev.yml
environment-name: pyjs-wasm
micromamba-version: "1.4.1"

- name: Setup emsdk
run: |
micromamba activate pyjs-wasm
emsdk install ${{matrix.emsdk_ver}}
- name: Build node-tests
run: |
micromamba activate pyjs-wasm
python -m pip install git+https://github.com/emscripten-forge/empack@main --no-deps --ignore-installed
emsdk activate ${{matrix.emsdk_ver}}
source $CONDA_EMSDK_DIR/emsdk_env.sh
micromamba create -n pyjs-build-wasm \
--platform=emscripten-32 \
-c https://repo.mamba.pm/emscripten-forge \
-c https://repo.mamba.pm/conda-forge \
--yes \
python pybind11 nlohmann_json pybind11_json numpy pytest bzip2 sqlite zlib libffi
mkdir build
pushd build
export PREFIX=$MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm
export CMAKE_PREFIX_PATH=$PREFIX
export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX
# build pyjs
emcmake cmake \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \
-DBUILD_RUNTIME_BROWSER=OFF \
-DBUILD_RUNTIME_NODE=ON \
-DWITH_NODE_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=$PREFIX \
..
make -j2
make install
popd
- name: Test in nodejs
run: |
micromamba activate pyjs-wasm
pyjs_code_runner run script \
node \
--conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm \
--mount $(pwd)/tests:tests \
--script main.py \
--work-dir /tests \
--pyjs-dir $(pwd)/build \
--node-binary /home/runner/micromamba-root/envs/pyjs-wasm/bin/node \
--async-main
62 changes: 41 additions & 21 deletions include/pyjs/pre_js/dynload/dynload.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,13 @@ const python_version = {
major: 3,
minor: 10
};
const API = {
python_version: python_version,
defaultLdLibraryPath: ["/lib", "/usr/lib", "/usr/local/lib"],
sitepackages: `/lib/python${python_version.major}.${python_version.minor}/site-packages`,
}

const SHARED_LIB_LOCATIONS = new Set()
// for each element in defaultLdLibraryPath
for (const path of API.defaultLdLibraryPath) {
SHARED_LIB_LOCATIONS.add(path);
}







const memoize = (fn) => {
let cache = {};
Expand Down Expand Up @@ -42,10 +38,15 @@ function createLock() {
return acquireLock;
}

function isInSharedLibraryPath(libPath){
function isInSharedLibraryPath(prefix, libPath){
if (libPath.startsWith("/")){
const dirname = libPath.substring(0, libPath.lastIndexOf("/"));
return SHARED_LIB_LOCATIONS.has(dirname);
if(prefix == "/"){
return (dirname == `$/lib`);
}
else{
return (dirname == `${prefix}/lib`);
}
}
else{
return false;
Expand All @@ -54,14 +55,25 @@ function isInSharedLibraryPath(libPath){


async function loadDynlibsFromPackage(
prefix,
pkg_file_name,
pkg_is_shared_library,
dynlibPaths,
) {

// for(const path of dynlibPaths){
// console.log(path);
// }

// assume that shared libraries of a package are located in <package-name>.libs directory,
// following the convention of auditwheel.
const auditWheelLibDir = `${API.sitepackages}/${
if(prefix == "/"){
var sitepackages = `/lib/python${python_version.major}.${python_version.minor}/site-packages`
}
else{
var sitepackages = `${prefix}/lib/python${python_version.major}.${python_version.minor}/site-packages`
}
const auditWheelLibDir = `${sitepackages}/${
pkg_file_name.split("-")[0]
}.libs`;

Expand All @@ -70,7 +82,7 @@ async function loadDynlibsFromPackage(

const forceGlobal = !!pkg_is_shared_library;

//console.log("forceGlobal", forceGlobal,"pkg_is_shared_library", pkg_is_shared_library);


let dynlibs = [];

Expand All @@ -92,7 +104,7 @@ async function loadDynlibsFromPackage(
//console.log(`isInSharedLibraryPath ${path} ${isInSharedLibraryPath(path)}`);
return {
path: path,
global: global || !! pkg_is_shared_library || isInSharedLibraryPath(path) || path.startsWith(auditWheelLibDir),
global: global || !! pkg_is_shared_library || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir),
};
});
}
Expand All @@ -101,19 +113,27 @@ async function loadDynlibsFromPackage(

for (const { path, global } of dynlibs) {
//console.log(`loading dynlib ${path} global ${global}`);
await loadDynlib(path, global, [auditWheelLibDir], readFileMemoized);
await loadDynlib(prefix, path, global, [auditWheelLibDir], readFileMemoized);
}
}

function createDynlibFS(
prefix,
lib,
searchDirs,
readFileFunc
) {
const dirname = lib.substring(0, lib.lastIndexOf("/"));

let _searchDirs = searchDirs || [];
_searchDirs = _searchDirs.concat([dirname], API.defaultLdLibraryPath,);

if(prefix == "/"){
_searchDirs = _searchDirs.concat([dirname], [`/lib`]);
}
else{
_searchDirs = _searchDirs.concat([dirname], [`${prefix}/lib`]);
}


const resolvePath = (path) => {
//console.log("resolvePath", path);
Expand All @@ -126,6 +146,7 @@ function createDynlibFS(
const fullPath = Module.PATH.join2(dir, path);
//console.log("SERARCHING", fullPath);
if (Module.FS.findObject(fullPath) !== null) {
//console.log("FOUND", fullPath);
return fullPath;
}
}
Expand Down Expand Up @@ -184,18 +205,17 @@ function calculateGlobalLibs(
// it.
const acquireDynlibLock = createLock();

async function loadDynlib(lib, global, searchDirs, readFileFunc) {
async function loadDynlib(prefix, lib, global, searchDirs, readFileFunc) {
if (searchDirs === undefined) {
searchDirs = [];
}
const releaseDynlibLock = await acquireDynlibLock();

try {
const fs = createDynlibFS(lib, searchDirs, readFileFunc);
const fs = createDynlibFS(prefix, lib, searchDirs, readFileFunc);

const libName = Module.PATH.basename(lib);
//console.log("load lib", lib);
//console.log(`libName ${libName}`)
//console.log(`load ${lib} (${libName}) `)

await Module.loadDynamicLibrary(libName, {
loadAsync: true,
Expand Down
80 changes: 26 additions & 54 deletions include/pyjs/pre_js/fetch.js
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,30 @@



async function fetchByteArray(url){
let response = await fetch(url)
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
let arrayBuffer = await response.arrayBuffer()
let byte_array = new Uint8Array(arrayBuffer)
return byte_array
}

Module["_fetch_byte_array"] = fetchByteArray;


async function fetchJson(url){
let response = await fetch(url)
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
let json = await response.json()
return json
}



Module["_parallel_fetch_array_buffer"] = async function (urls){
let promises = urls.map(url => fetch(url).then(response => response.arrayBuffer()));
return await Promise.all(promises);
Expand Down Expand Up @@ -96,58 +122,4 @@ Module["_parallel_fetch_arraybuffers_with_progress_bar"] = async function (urls,
return fetch_arraybuffer_with_progress_bar(url,index, report_total_length,report_progress, report_finished)
})
return await Promise.all(futures);
}


Module["fetch_and_untar"] = async function(
urls,
filenames,
extract_to,
verbose=false,
){
let log = function(...args){
if(verbose){
console.log(...args)
}
}

let progress_callback = undefined
if (verbose){
progress_callback = function(n_bytes, total_bytes, ...args){
console.log(`fetching ${n_bytes} of ${total_bytes} bytes`)
}
}

let shared_libs = new Array(urls.length)

let done_callback = async function(i, byte_array){
log("finished download",filenames[i])

// get dirname from filename
let dirname = filenames[i].split("/").slice(0,-1).join("/")
Module.mkdirs(dirname);

// get filename without directory
let filename = filenames[i].split("/").slice(-1)[0]


log("writing",filenames[i])
Module.FS.writeFile(filenames[i], byte_array);

log("untar", filenames[i])
shared_libs[i] = Module._untar(filenames[i], extract_to[i]);
}

await Module._parallel_fetch_arraybuffers_with_progress_bar(urls,done_callback, progress_callback)


for(let i=0;i<urls.length;i++){
if(shared_libs[i].length > 0){
await Module._loadDynlibsFromPackage(
filenames[i],
false,
shared_libs[i]
)
}
}
}
Loading

0 comments on commit 5ef613d

Please sign in to comment.